arch/x86/platform/efi/efi.c

   1 /*
   2  * Common EFI (Extensible Firmware Interface) support functions
   3  * Based on Extensible Firmware Interface Specification version 1.0
   4  *
   5  * Copyright (C) 1999 VA Linux Systems
   6  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
   7  * Copyright (C) 1999-2002 Hewlett-Packard Co.
   8  *      David Mosberger-Tang <davidm@hpl.hp.com>
   9  *      Stephane Eranian <eranian@hpl.hp.com>
  10  * Copyright (C) 2005-2008 Intel Co.
  11  *      Fenghua Yu <fenghua.yu@intel.com>
  12  *      Bibo Mao <bibo.mao@intel.com>
  13  *      Chandramouli Narayanan <mouli@linux.intel.com>
  14  *      Huang Ying <ying.huang@intel.com>
  15  * Copyright (C) 2013 SuSE Labs
  16  *      Borislav Petkov <bp@suse.de> - runtime services VA mapping
  17  *
  18  * Copied from efi_32.c to eliminate the duplicated code between EFI
  19  * 32/64 support code. --ying 2007-10-26
  20  *
  21  * All EFI Runtime Services are not implemented yet as EFI only
  22  * supports physical mode addressing on SoftSDV. This is to be fixed
  23  * in a future version.  --drummond 1999-07-20
  24  *
  25  * Implemented EFI runtime services and virtual mode calls.  --davidm
  26  *
  27  * Goutham Rao: <goutham.rao@intel.com>
  28  *      Skip non-WB memory and ignore empty memory ranges.
  29  */
  30
  31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  32
  33 #include <linux/kernel.h>
  34 #include <linux/init.h>
  35 #include <linux/efi.h>
  36 #include <linux/efi-bgrt.h>
  37 #include <linux/export.h>
  38 #include <linux/bootmem.h>
  39 #include <linux/slab.h>
  40 #include <linux/memblock.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/time.h>
  44 #include <linux/io.h>
  45 #include <linux/reboot.h>
  46 #include <linux/bcd.h>
  47
  48 #include <asm/setup.h>
  49 #include <asm/efi.h>
  50 #include <asm/time.h>
  51 #include <asm/cacheflush.h>
  52 #include <asm/tlbflush.h>
  53 #include <asm/x86_init.h>
  54 #include <asm/rtc.h>
  55 #include <asm/uv/uv.h>
  56
  57 #define EFI_DEBUG
  58
  59 struct efi_memory_map memmap;
  60
  61 static struct efi efi_phys __initdata;
  62 static efi_system_table_t efi_systab __initdata;
  63
  64 static efi_config_table_type_t arch_tables[] __initdata = {
  65 #ifdef CONFIG_X86_UV
  66         {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab},
  67 #endif
  68         {NULL_GUID, NULL, NULL},
  69 };
  70
  71 u64 efi_setup;          /* efi setup_data physical address */
  72
  73 static int add_efi_memmap __initdata;
  74 static int __init setup_add_efi_memmap(char *arg)
  75 {
  76         add_efi_memmap = 1;
  77         return 0;
  78 }
  79 early_param("add_efi_memmap", setup_add_efi_memmap);
  80
  81 static efi_status_t __init phys_efi_set_virtual_address_map(
  82         unsigned long memory_map_size,
  83         unsigned long descriptor_size,
  84         u32 descriptor_version,
  85         efi_memory_desc_t *virtual_map)
  86 {
  87         efi_status_t status;
  88
  89         efi_call_phys_prolog();
  90         status = efi_call_phys(efi_phys.set_virtual_address_map,
  91                                memory_map_size, descriptor_size,
  92                                descriptor_version, virtual_map);
  93         efi_call_phys_epilog();
  94         return status;
  95 }
  96
  97 void efi_get_time(struct timespec *now)
  98 {
  99         efi_status_t status;
 100         efi_time_t eft;
 101         efi_time_cap_t cap;
 102
 103         status = efi.get_time(&eft, &cap);
 104         if (status != EFI_SUCCESS)
 105                 pr_err("Oops: efitime: can't read time!\n");
 106
 107         now->tv_sec = mktime(eft.year, eft.month, eft.day, eft.hour,
 108                              eft.minute, eft.second);
 109         now->tv_nsec = 0;
 110 }
 111
 112 /*
 113  * Tell the kernel about the EFI memory map.  This might include
 114  * more than the max 128 entries that can fit in the e820 legacy
 115  * (zeropage) memory map.
 116  */
 117
 118 static void __init do_add_efi_memmap(void)
 119 {
 120         void *p;
 121
 122         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 123                 efi_memory_desc_t *md = p;
 124                 unsigned long long start = md->phys_addr;
 125                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 126                 int e820_type;
 127
 128                 switch (md->type) {
 129                 case EFI_LOADER_CODE:
 130                 case EFI_LOADER_DATA:
 131                 case EFI_BOOT_SERVICES_CODE:
 132                 case EFI_BOOT_SERVICES_DATA:
 133                 case EFI_CONVENTIONAL_MEMORY:
 134                         if (md->attribute & EFI_MEMORY_WB)
 135                                 e820_type = E820_RAM;
 136                         else
 137                                 e820_type = E820_RESERVED;
 138                         break;
 139                 case EFI_ACPI_RECLAIM_MEMORY:
 140                         e820_type = E820_ACPI;
 141                         break;
 142                 case EFI_ACPI_MEMORY_NVS:
 143                         e820_type = E820_NVS;
 144                         break;
 145                 case EFI_UNUSABLE_MEMORY:
 146                         e820_type = E820_UNUSABLE;
 147                         break;
 148                 default:
 149                         /*
 150                          * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
 151                          * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
 152                          * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
 153                          */
 154                         e820_type = E820_RESERVED;
 155                         break;
 156                 }
 157                 e820_add_region(start, size, e820_type);
 158         }
 159         sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
 160 }
 161
 162 int __init efi_memblock_x86_reserve_range(void)
 163 {
 164         struct efi_info *e = &boot_params.efi_info;
 165         unsigned long pmap;
 166
 167         if (efi_enabled(EFI_PARAVIRT))
 168                 return 0;
 169
 170 #ifdef CONFIG_X86_32
 171         /* Can't handle data above 4GB at this time */
 172         if (e->efi_memmap_hi) {
 173                 pr_err("Memory map is above 4GB, disabling EFI.\n");
 174                 return -EINVAL;
 175         }
 176         pmap =  e->efi_memmap;
 177 #else
 178         pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
 179 #endif
 180         memmap.phys_map         = (void *)pmap;
 181         memmap.nr_map           = e->efi_memmap_size /
 182                                   e->efi_memdesc_size;
 183         memmap.desc_size        = e->efi_memdesc_size;
 184         memmap.desc_version     = e->efi_memdesc_version;
 185
 186         memblock_reserve(pmap, memmap.nr_map * memmap.desc_size);
 187
 188         efi.memmap = &memmap;
 189
 190         return 0;
 191 }
 192
 193 static void __init print_efi_memmap(void)
 194 {
 195 #ifdef EFI_DEBUG
 196         efi_memory_desc_t *md;
 197         void *p;
 198         int i;
 199
 200         for (p = memmap.map, i = 0;
 201              p < memmap.map_end;
 202              p += memmap.desc_size, i++) {
 203                 char buf[64];
 204
 205                 md = p;
 206                 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx) (%lluMB)\n",
 207                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 208                         md->phys_addr,
 209                         md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT),
 210                         (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
 211         }
 212 #endif  /*  EFI_DEBUG  */
 213 }
 214
 215 void __init efi_unmap_memmap(void)
 216 {
 217         clear_bit(EFI_MEMMAP, &efi.flags);
 218         if (memmap.map) {
 219                 early_memunmap(memmap.map, memmap.nr_map * memmap.desc_size);
 220                 memmap.map = NULL;
 221         }
 222 }
 223
 224 static int __init efi_systab_init(void *phys)
 225 {
 226         if (efi_enabled(EFI_64BIT)) {
 227                 efi_system_table_64_t *systab64;
 228                 struct efi_setup_data *data = NULL;
 229                 u64 tmp = 0;
 230
 231                 if (efi_setup) {
 232                         data = early_memremap(efi_setup, sizeof(*data));
 233                         if (!data)
 234                                 return -ENOMEM;
 235                 }
 236                 systab64 = early_memremap((unsigned long)phys,
 237                                          sizeof(*systab64));
 238                 if (systab64 == NULL) {
 239                         pr_err("Couldn't map the system table!\n");
 240                         if (data)
 241                                 early_memunmap(data, sizeof(*data));
 242                         return -ENOMEM;
 243                 }
 244
 245                 efi_systab.hdr = systab64->hdr;
 246                 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
 247                                               systab64->fw_vendor;
 248                 tmp |= data ? data->fw_vendor : systab64->fw_vendor;
 249                 efi_systab.fw_revision = systab64->fw_revision;
 250                 efi_systab.con_in_handle = systab64->con_in_handle;
 251                 tmp |= systab64->con_in_handle;
 252                 efi_systab.con_in = systab64->con_in;
 253                 tmp |= systab64->con_in;
 254                 efi_systab.con_out_handle = systab64->con_out_handle;
 255                 tmp |= systab64->con_out_handle;
 256                 efi_systab.con_out = systab64->con_out;
 257                 tmp |= systab64->con_out;
 258                 efi_systab.stderr_handle = systab64->stderr_handle;
 259                 tmp |= systab64->stderr_handle;
 260                 efi_systab.stderr = systab64->stderr;
 261                 tmp |= systab64->stderr;
 262                 efi_systab.runtime = data ?
 263                                      (void *)(unsigned long)data->runtime :
 264                                      (void *)(unsigned long)systab64->runtime;
 265                 tmp |= data ? data->runtime : systab64->runtime;
 266                 efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
 267                 tmp |= systab64->boottime;
 268                 efi_systab.nr_tables = systab64->nr_tables;
 269                 efi_systab.tables = data ? (unsigned long)data->tables :
 270                                            systab64->tables;
 271                 tmp |= data ? data->tables : systab64->tables;
 272
 273                 early_memunmap(systab64, sizeof(*systab64));
 274                 if (data)
 275                         early_memunmap(data, sizeof(*data));
 276 #ifdef CONFIG_X86_32
 277                 if (tmp >> 32) {
 278                         pr_err("EFI data located above 4GB, disabling EFI.\n");
 279                         return -EINVAL;
 280                 }
 281 #endif
 282         } else {
 283                 efi_system_table_32_t *systab32;
 284
 285                 systab32 = early_memremap((unsigned long)phys,
 286                                          sizeof(*systab32));
 287                 if (systab32 == NULL) {
 288                         pr_err("Couldn't map the system table!\n");
 289                         return -ENOMEM;
 290                 }
 291
 292                 efi_systab.hdr = systab32->hdr;
 293                 efi_systab.fw_vendor = systab32->fw_vendor;
 294                 efi_systab.fw_revision = systab32->fw_revision;
 295                 efi_systab.con_in_handle = systab32->con_in_handle;
 296                 efi_systab.con_in = systab32->con_in;
 297                 efi_systab.con_out_handle = systab32->con_out_handle;
 298                 efi_systab.con_out = systab32->con_out;
 299                 efi_systab.stderr_handle = systab32->stderr_handle;
 300                 efi_systab.stderr = systab32->stderr;
 301                 efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
 302                 efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
 303                 efi_systab.nr_tables = systab32->nr_tables;
 304                 efi_systab.tables = systab32->tables;
 305
 306                 early_memunmap(systab32, sizeof(*systab32));
 307         }
 308
 309         efi.systab = &efi_systab;
 310
 311         /*
 312          * Verify the EFI Table
 313          */
 314         if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
 315                 pr_err("System table signature incorrect!\n");
 316                 return -EINVAL;
 317         }
 318         if ((efi.systab->hdr.revision >> 16) == 0)
 319                 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
 320                        efi.systab->hdr.revision >> 16,
 321                        efi.systab->hdr.revision & 0xffff);
 322
 323         set_bit(EFI_SYSTEM_TABLES, &efi.flags);
 324
 325         return 0;
 326 }
 327
 328 static int __init efi_runtime_init32(void)
 329 {
 330         efi_runtime_services_32_t *runtime;
 331
 332         runtime = early_memremap((unsigned long)efi.systab->runtime,
 333                         sizeof(efi_runtime_services_32_t));
 334         if (!runtime) {
 335                 pr_err("Could not map the runtime service table!\n");
 336                 return -ENOMEM;
 337         }
 338
 339         /*
 340          * We will only need *early* access to the SetVirtualAddressMap
 341          * EFI runtime service. All other runtime services will be called
 342          * via the virtual mapping.
 343          */
 344         efi_phys.set_virtual_address_map =
 345                         (efi_set_virtual_address_map_t *)
 346                         (unsigned long)runtime->set_virtual_address_map;
 347         early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
 348
 349         return 0;
 350 }
 351
 352 static int __init efi_runtime_init64(void)
 353 {
 354         efi_runtime_services_64_t *runtime;
 355
 356         runtime = early_memremap((unsigned long)efi.systab->runtime,
 357                         sizeof(efi_runtime_services_64_t));
 358         if (!runtime) {
 359                 pr_err("Could not map the runtime service table!\n");
 360                 return -ENOMEM;
 361         }
 362
 363         /*
 364          * We will only need *early* access to the SetVirtualAddressMap
 365          * EFI runtime service. All other runtime services will be called
 366          * via the virtual mapping.
 367          */
 368         efi_phys.set_virtual_address_map =
 369                         (efi_set_virtual_address_map_t *)
 370                         (unsigned long)runtime->set_virtual_address_map;
 371         early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
 372
 373         return 0;
 374 }
 375
 376 static int __init efi_runtime_init(void)
 377 {
 378         int rv;
 379
 380         /*
 381          * Check out the runtime services table. We need to map
 382          * the runtime services table so that we can grab the physical
 383          * address of several of the EFI runtime functions, needed to
 384          * set the firmware into virtual mode.
 385          *
 386          * When EFI_PARAVIRT is in force then we could not map runtime
 387          * service memory region because we do not have direct access to it.
 388          * However, runtime services are available through proxy functions
 389          * (e.g. in case of Xen dom0 EFI implementation they call special
 390          * hypercall which executes relevant EFI functions) and that is why
 391          * they are always enabled.
 392          */
 393
 394         if (!efi_enabled(EFI_PARAVIRT)) {
 395                 if (efi_enabled(EFI_64BIT))
 396                         rv = efi_runtime_init64();
 397                 else
 398                         rv = efi_runtime_init32();
 399
 400                 if (rv)
 401                         return rv;
 402         }
 403
 404         set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 405
 406         return 0;
 407 }
 408
 409 static int __init efi_memmap_init(void)
 410 {
 411         if (efi_enabled(EFI_PARAVIRT))
 412                 return 0;
 413
 414         /* Map the EFI memory map */
 415         memmap.map = early_memremap((unsigned long)memmap.phys_map,
 416                                    memmap.nr_map * memmap.desc_size);
 417         if (memmap.map == NULL) {
 418                 pr_err("Could not map the memory map!\n");
 419                 return -ENOMEM;
 420         }
 421         memmap.map_end = memmap.map + (memmap.nr_map * memmap.desc_size);
 422
 423         if (add_efi_memmap)
 424                 do_add_efi_memmap();
 425
 426         set_bit(EFI_MEMMAP, &efi.flags);
 427
 428         return 0;
 429 }
 430
 431 void __init efi_init(void)
 432 {
 433         efi_char16_t *c16;
 434         char vendor[100] = "unknown";
 435         int i = 0;
 436         void *tmp;
 437
 438 #ifdef CONFIG_X86_32
 439         if (boot_params.efi_info.efi_systab_hi ||
 440             boot_params.efi_info.efi_memmap_hi) {
 441                 pr_info("Table located above 4GB, disabling EFI.\n");
 442                 return;
 443         }
 444         efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
 445 #else
 446         efi_phys.systab = (efi_system_table_t *)
 447                           (boot_params.efi_info.efi_systab |
 448                           ((__u64)boot_params.efi_info.efi_systab_hi<<32));
 449 #endif
 450
 451         if (efi_systab_init(efi_phys.systab))
 452                 return;
 453
 454         efi.config_table = (unsigned long)efi.systab->tables;
 455         efi.fw_vendor    = (unsigned long)efi.systab->fw_vendor;
 456         efi.runtime      = (unsigned long)efi.systab->runtime;
 457
 458         /*
 459          * Show what we know for posterity
 460          */
 461         c16 = tmp = early_memremap(efi.systab->fw_vendor, 2);
 462         if (c16) {
 463                 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i)
 464                         vendor[i] = *c16++;
 465                 vendor[i] = '\0';
 466         } else
 467                 pr_err("Could not map the firmware vendor!\n");
 468         early_memunmap(tmp, 2);
 469
 470         pr_info("EFI v%u.%.02u by %s\n",
 471                 efi.systab->hdr.revision >> 16,
 472                 efi.systab->hdr.revision & 0xffff, vendor);
 473
 474         if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
 475                 return;
 476
 477         if (efi_config_init(arch_tables))
 478                 return;
 479
 480         /*
 481          * Note: We currently don't support runtime services on an EFI
 482          * that doesn't match the kernel 32/64-bit mode.
 483          */
 484
 485         if (!efi_runtime_supported())
 486                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
 487         else {
 488                 if (efi_runtime_disabled() || efi_runtime_init())
 489                         return;
 490         }
 491         if (efi_memmap_init())
 492                 return;
 493
 494         if (efi_enabled(EFI_DBG))
 495                 print_efi_memmap();
 496 }
 497
 498 void __init efi_late_init(void)
 499 {
 500         efi_bgrt_init();
 501 }
 502
 503 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
 504 {
 505         u64 addr, npages;
 506
 507         addr = md->virt_addr;
 508         npages = md->num_pages;
 509
 510         memrange_efi_to_native(&addr, &npages);
 511
 512         if (executable)
 513                 set_memory_x(addr, npages);
 514         else
 515                 set_memory_nx(addr, npages);
 516 }
 517
 518 void __init runtime_code_page_mkexec(void)
 519 {
 520         efi_memory_desc_t *md;
 521         void *p;
 522
 523         /* Make EFI runtime service code area executable */
 524         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 525                 md = p;
 526
 527                 if (md->type != EFI_RUNTIME_SERVICES_CODE)
 528                         continue;
 529
 530                 efi_set_executable(md, true);
 531         }
 532 }
 533
 534 void __init efi_memory_uc(u64 addr, unsigned long size)
 535 {
 536         unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
 537         u64 npages;
 538
 539         npages = round_up(size, page_shift) / page_shift;
 540         memrange_efi_to_native(&addr, &npages);
 541         set_memory_uc(addr, npages);
 542 }
 543
 544 void __init old_map_region(efi_memory_desc_t *md)
 545 {
 546         u64 start_pfn, end_pfn, end;
 547         unsigned long size;
 548         void *va;
 549
 550         start_pfn = PFN_DOWN(md->phys_addr);
 551         size      = md->num_pages << PAGE_SHIFT;
 552         end       = md->phys_addr + size;
 553         end_pfn   = PFN_UP(end);
 554
 555         if (pfn_range_is_mapped(start_pfn, end_pfn)) {
 556                 va = __va(md->phys_addr);
 557
 558                 if (!(md->attribute & EFI_MEMORY_WB))
 559                         efi_memory_uc((u64)(unsigned long)va, size);
 560         } else
 561                 va = efi_ioremap(md->phys_addr, size,
 562                                  md->type, md->attribute);
 563
 564         md->virt_addr = (u64) (unsigned long) va;
 565         if (!va)
 566                 pr_err("ioremap of 0x%llX failed!\n",
 567                        (unsigned long long)md->phys_addr);
 568 }
 569
 570 /* Merge contiguous regions of the same type and attribute */
 571 static void __init efi_merge_regions(void)
 572 {
 573         void *p;
 574         efi_memory_desc_t *md, *prev_md = NULL;
 575
 576         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 577                 u64 prev_size;
 578                 md = p;
 579
 580                 if (!prev_md) {
 581                         prev_md = md;
 582                         continue;
 583                 }
 584
 585                 if (prev_md->type != md->type ||
 586                     prev_md->attribute != md->attribute) {
 587                         prev_md = md;
 588                         continue;
 589                 }
 590
 591                 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
 592
 593                 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
 594                         prev_md->num_pages += md->num_pages;
 595                         md->type = EFI_RESERVED_TYPE;
 596                         md->attribute = 0;
 597                         continue;
 598                 }
 599                 prev_md = md;
 600         }
 601 }
 602
 603 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
 604 {
 605         unsigned long size;
 606         u64 end, systab;
 607
 608         size = md->num_pages << EFI_PAGE_SHIFT;
 609         end = md->phys_addr + size;
 610         systab = (u64)(unsigned long)efi_phys.systab;
 611         if (md->phys_addr <= systab && systab < end) {
 612                 systab += md->virt_addr - md->phys_addr;
 613                 efi.systab = (efi_system_table_t *)(unsigned long)systab;
 614         }
 615 }
 616
 617 static void __init save_runtime_map(void)
 618 {
 619 #ifdef CONFIG_KEXEC
 620         efi_memory_desc_t *md;
 621         void *tmp, *p, *q = NULL;
 622         int count = 0;
 623
 624         if (efi_enabled(EFI_OLD_MEMMAP))
 625                 return;
 626
 627         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 628                 md = p;
 629
 630                 if (!(md->attribute & EFI_MEMORY_RUNTIME) ||
 631                     (md->type == EFI_BOOT_SERVICES_CODE) ||
 632                     (md->type == EFI_BOOT_SERVICES_DATA))
 633                         continue;
 634                 tmp = krealloc(q, (count + 1) * memmap.desc_size, GFP_KERNEL);
 635                 if (!tmp)
 636                         goto out;
 637                 q = tmp;
 638
 639                 memcpy(q + count * memmap.desc_size, md, memmap.desc_size);
 640                 count++;
 641         }
 642
 643         efi_runtime_map_setup(q, count, memmap.desc_size);
 644         return;
 645
 646 out:
 647         kfree(q);
 648         pr_err("Error saving runtime map, efi runtime on kexec non-functional!!\n");
 649 #endif
 650 }
 651
 652 static void *realloc_pages(void *old_memmap, int old_shift)
 653 {
 654         void *ret;
 655
 656         ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
 657         if (!ret)
 658                 goto out;
 659
 660         /*
 661          * A first-time allocation doesn't have anything to copy.
 662          */
 663         if (!old_memmap)
 664                 return ret;
 665
 666         memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
 667
 668 out:
 669         free_pages((unsigned long)old_memmap, old_shift);
 670         return ret;
 671 }
 672
 673 /*
 674  * Map the efi memory ranges of the runtime services and update new_mmap with
 675  * virtual addresses.
 676  */
 677 static void * __init efi_map_regions(int *count, int *pg_shift)
 678 {
 679         void *p, *new_memmap = NULL;
 680         unsigned long left = 0;
 681         efi_memory_desc_t *md;
 682
 683         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 684                 md = p;
 685                 if (!(md->attribute & EFI_MEMORY_RUNTIME)) {
 686 #ifdef CONFIG_X86_64
 687                         if (md->type != EFI_BOOT_SERVICES_CODE &&
 688                             md->type != EFI_BOOT_SERVICES_DATA)
 689 #endif
 690                                 continue;
 691                 }
 692
 693                 efi_map_region(md);
 694                 get_systab_virt_addr(md);
 695
 696                 if (left < memmap.desc_size) {
 697                         new_memmap = realloc_pages(new_memmap, *pg_shift);
 698                         if (!new_memmap)
 699                                 return NULL;
 700
 701                         left += PAGE_SIZE << *pg_shift;
 702                         (*pg_shift)++;
 703                 }
 704
 705                 memcpy(new_memmap + (*count * memmap.desc_size), md,
 706                        memmap.desc_size);
 707
 708                 left -= memmap.desc_size;
 709                 (*count)++;
 710         }
 711
 712         return new_memmap;
 713 }
 714
 715 static void __init kexec_enter_virtual_mode(void)
 716 {
 717 #ifdef CONFIG_KEXEC
 718         efi_memory_desc_t *md;
 719         void *p;
 720
 721         efi.systab = NULL;
 722
 723         /*
 724          * We don't do virtual mode, since we don't do runtime services, on
 725          * non-native EFI
 726          */
 727         if (!efi_is_native()) {
 728                 efi_unmap_memmap();
 729                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 730                 return;
 731         }
 732
 733         /*
 734         * Map efi regions which were passed via setup_data. The virt_addr is a
 735         * fixed addr which was used in first kernel of a kexec boot.
 736         */
 737         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 738                 md = p;
 739                 efi_map_region_fixed(md); /* FIXME: add error handling */
 740                 get_systab_virt_addr(md);
 741         }
 742
 743         save_runtime_map();
 744
 745         BUG_ON(!efi.systab);
 746
 747         efi_sync_low_kernel_mappings();
 748
 749         /*
 750          * Now that EFI is in virtual mode, update the function
 751          * pointers in the runtime service table to the new virtual addresses.
 752          *
 753          * Call EFI services through wrapper functions.
 754          */
 755         efi.runtime_version = efi_systab.hdr.revision;
 756
 757         efi_native_runtime_setup();
 758
 759         efi.set_virtual_address_map = NULL;
 760
 761         if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
 762                 runtime_code_page_mkexec();
 763
 764         /* clean DUMMY object */
 765         efi_delete_dummy_variable();
 766 #endif
 767 }
 768
 769 /*
 770  * This function will switch the EFI runtime services to virtual mode.
 771  * Essentially, we look through the EFI memmap and map every region that
 772  * has the runtime attribute bit set in its memory descriptor into the
 773  * ->trampoline_pgd page table using a top-down VA allocation scheme.
 774  *
 775  * The old method which used to update that memory descriptor with the
 776  * virtual address obtained from ioremap() is still supported when the
 777  * kernel is booted with efi=old_map on its command line. Same old
 778  * method enabled the runtime services to be called without having to
 779  * thunk back into physical mode for every invocation.
 780  *
 781  * The new method does a pagetable switch in a preemption-safe manner
 782  * so that we're in a different address space when calling a runtime
 783  * function. For function arguments passing we do copy the PGDs of the
 784  * kernel page table into ->trampoline_pgd prior to each call.
 785  *
 786  * Specially for kexec boot, efi runtime maps in previous kernel should
 787  * be passed in via setup_data. In that case runtime ranges will be mapped
 788  * to the same virtual addresses as the first kernel, see
 789  * kexec_enter_virtual_mode().
 790  */
 791 static void __init __efi_enter_virtual_mode(void)
 792 {
 793         int count = 0, pg_shift = 0;
 794         void *new_memmap = NULL;
 795         efi_status_t status;
 796
 797         efi.systab = NULL;
 798
 799         efi_merge_regions();
 800         new_memmap = efi_map_regions(&count, &pg_shift);
 801         if (!new_memmap) {
 802                 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
 803                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 804                 return;
 805         }
 806
 807         save_runtime_map();
 808
 809         BUG_ON(!efi.systab);
 810
 811         if (efi_setup_page_tables(__pa(new_memmap), 1 << pg_shift)) {
 812                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 813                 return;
 814         }
 815
 816         efi_sync_low_kernel_mappings();
 817         efi_dump_pagetable();
 818
 819         if (efi_is_native()) {
 820                 status = phys_efi_set_virtual_address_map(
 821                                 memmap.desc_size * count,
 822                                 memmap.desc_size,
 823                                 memmap.desc_version,
 824                                 (efi_memory_desc_t *)__pa(new_memmap));
 825         } else {
 826                 status = efi_thunk_set_virtual_address_map(
 827                                 efi_phys.set_virtual_address_map,
 828                                 memmap.desc_size * count,
 829                                 memmap.desc_size,
 830                                 memmap.desc_version,
 831                                 (efi_memory_desc_t *)__pa(new_memmap));
 832         }
 833
 834         if (status != EFI_SUCCESS) {
 835                 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
 836                          status);
 837                 panic("EFI call to SetVirtualAddressMap() failed!");
 838         }
 839
 840         /*
 841          * Now that EFI is in virtual mode, update the function
 842          * pointers in the runtime service table to the new virtual addresses.
 843          *
 844          * Call EFI services through wrapper functions.
 845          */
 846         efi.runtime_version = efi_systab.hdr.revision;
 847
 848         if (efi_is_native())
 849                 efi_native_runtime_setup();
 850         else
 851                 efi_thunk_runtime_setup();
 852
 853         efi.set_virtual_address_map = NULL;
 854
 855         efi_runtime_mkexec();
 856
 857         /*
 858          * We mapped the descriptor array into the EFI pagetable above but we're
 859          * not unmapping it here. Here's why:
 860          *
 861          * We're copying select PGDs from the kernel page table to the EFI page
 862          * table and when we do so and make changes to those PGDs like unmapping
 863          * stuff from them, those changes appear in the kernel page table and we
 864          * go boom.
 865          *
 866          * From setup_real_mode():
 867          *
 868          * ...
 869          * trampoline_pgd[0] = init_level4_pgt[pgd_index(__PAGE_OFFSET)].pgd;
 870          *
 871          * In this particular case, our allocation is in PGD 0 of the EFI page
 872          * table but we've copied that PGD from PGD[272] of the EFI page table:
 873          *
 874          *      pgd_index(__PAGE_OFFSET = 0xffff880000000000) = 272
 875          *
 876          * where the direct memory mapping in kernel space is.
 877          *
 878          * new_memmap's VA comes from that direct mapping and thus clearing it,
 879          * it would get cleared in the kernel page table too.
 880          *
 881          * efi_cleanup_page_tables(__pa(new_memmap), 1 << pg_shift);
 882          */
 883         free_pages((unsigned long)new_memmap, pg_shift);
 884
 885         /* clean DUMMY object */
 886         efi_delete_dummy_variable();
 887 }
 888
 889 void __init efi_enter_virtual_mode(void)
 890 {
 891         if (efi_enabled(EFI_PARAVIRT))
 892                 return;
 893
 894         if (efi_setup)
 895                 kexec_enter_virtual_mode();
 896         else
 897                 __efi_enter_virtual_mode();
 898 }
 899
 900 /*
 901  * Convenience functions to obtain memory types and attributes
 902  */
 903 u32 efi_mem_type(unsigned long phys_addr)
 904 {
 905         efi_memory_desc_t *md;
 906         void *p;
 907
 908         if (!efi_enabled(EFI_MEMMAP))
 909                 return 0;
 910
 911         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 912                 md = p;
 913                 if ((md->phys_addr <= phys_addr) &&
 914                     (phys_addr < (md->phys_addr +
 915                                   (md->num_pages << EFI_PAGE_SHIFT))))
 916                         return md->type;
 917         }
 918         return 0;
 919 }
 920
 921 u64 efi_mem_attributes(unsigned long phys_addr)
 922 {
 923         efi_memory_desc_t *md;
 924         void *p;
 925
 926         if (!efi_enabled(EFI_MEMMAP))
 927                 return 0;
 928
 929         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 930                 md = p;
 931                 if ((md->phys_addr <= phys_addr) &&
 932                     (phys_addr < (md->phys_addr +
 933                                   (md->num_pages << EFI_PAGE_SHIFT))))
 934                         return md->attribute;
 935         }
 936         return 0;
 937 }
 938
 939 static int __init arch_parse_efi_cmdline(char *str)
 940 {
 941         if (parse_option_str(str, "old_map"))
 942                 set_bit(EFI_OLD_MEMMAP, &efi.flags);
 943         if (parse_option_str(str, "debug"))
 944                 set_bit(EFI_DBG, &efi.flags);
 945
 946         return 0;
 947 }
 948 early_param("efi", arch_parse_efi_cmdline);