src/src/valgrind.h

   1 /* -*- c -*-
   2    ----------------------------------------------------------------
   3
   4    Notice that the following BSD-style license applies to this one
   5    file (valgrind.h) only.  The rest of Valgrind is licensed under the
   6    terms of the GNU General Public License, version 2, unless
   7    otherwise indicated.  See the COPYING file in the source
   8    distribution for details.
   9
  10    ----------------------------------------------------------------
  11
  12    This file is part of Valgrind, a dynamic binary instrumentation
  13    framework.
  14
  15    Copyright (C) 2000-2010 Julian Seward.  All rights reserved.
  16
  17    Redistribution and use in source and binary forms, with or without
  18    modification, are permitted provided that the following conditions
  19    are met:
  20
  21    1. Redistributions of source code must retain the above copyright
  22       notice, this list of conditions and the following disclaimer.
  23
  24    2. The origin of this software must not be misrepresented; you must
  25       not claim that you wrote the original software.  If you use this
  26       software in a product, an acknowledgment in the product
  27       documentation would be appreciated but is not required.
  28
  29    3. Altered source versions must be plainly marked as such, and must
  30       not be misrepresented as being the original software.
  31
  32    4. The name of the author may not be used to endorse or promote
  33       products derived from this software without specific prior written
  34       permission.
  35
  36    THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
  37    OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  38    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  39    ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
  40    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  41    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
  42    GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  43    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  44    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  45    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  46    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  47
  48    ----------------------------------------------------------------
  49
  50    Notice that the above BSD-style license applies to this one file
  51    (valgrind.h) only.  The entire rest of Valgrind is licensed under
  52    the terms of the GNU General Public License, version 2.  See the
  53    COPYING file in the source distribution for details.
  54
  55    ----------------------------------------------------------------
  56 */
  57
  58
  59 /* This file is for inclusion into client (your!) code.
  60
  61    You can use these macros to manipulate and query Valgrind's
  62    execution inside your own programs.
  63
  64    The resulting executables will still run without Valgrind, just a
  65    little bit more slowly than they otherwise would, but otherwise
  66    unchanged.  When not running on valgrind, each client request
  67    consumes very few (eg. 7) instructions, so the resulting performance
  68    loss is negligible unless you plan to execute client requests
  69    millions of times per second.  Nevertheless, if that is still a
  70    problem, you can compile with the NVALGRIND symbol defined (gcc
  71    -DNVALGRIND) so that client requests are not even compiled in.  */
  72
  73 #ifndef __VALGRIND_H
  74 #define __VALGRIND_H
  75
  76
  77 /* ------------------------------------------------------------------ */
  78 /* VERSION NUMBER OF VALGRIND                                         */
  79 /* ------------------------------------------------------------------ */
  80
  81 /* Specify Valgrind's version number, so that user code can
  82    conditionally compile based on our version number.  Note that these
  83    were introduced at version 3.6 and so do not exist in version 3.5
  84    or earlier.  The recommended way to use them to check for "version
  85    X.Y or later" is (eg)
  86
  87 #if defined(__VALGRIND_MAJOR__) && defined(__VALGRIND_MINOR__)   \
  88     && (__VALGRIND_MAJOR__ > 3                                   \
  89         || (__VALGRIND_MAJOR__ == 3 && __VALGRIND_MINOR__ >= 6))
  90 */
  91 #define __VALGRIND_MAJOR__    3
  92 #define __VALGRIND_MINOR__    6
  93
  94
  95 #include <stdarg.h>
  96
  97 /* Nb: this file might be included in a file compiled with -ansi.  So
  98    we can't use C++ style "//" comments nor the "asm" keyword (instead
  99    use "__asm__"). */
 100
 101 /* Derive some tags indicating what the target platform is.  Note
 102    that in this file we're using the compiler's CPP symbols for
 103    identifying architectures, which are different to the ones we use
 104    within the rest of Valgrind.  Note, __powerpc__ is active for both
 105    32 and 64-bit PPC, whereas __powerpc64__ is only active for the
 106    latter (on Linux, that is).
 107
 108    Misc note: how to find out what's predefined in gcc by default:
 109    gcc -Wp,-dM somefile.c
 110 */
 111 #undef PLAT_ppc64_aix5
 112 #undef PLAT_ppc32_aix5
 113 #undef PLAT_x86_darwin
 114 #undef PLAT_amd64_darwin
 115 #undef PLAT_x86_win32
 116 #undef PLAT_x86_linux
 117 #undef PLAT_amd64_linux
 118 #undef PLAT_ppc32_linux
 119 #undef PLAT_ppc64_linux
 120 #undef PLAT_arm_linux
 121
 122 #if defined(_AIX) && defined(__64BIT__)
 123 #  define PLAT_ppc64_aix5 1
 124 #elif defined(_AIX) && !defined(__64BIT__)
 125 #  define PLAT_ppc32_aix5 1
 126 #elif defined(__APPLE__) && defined(__i386__)
 127 #  define PLAT_x86_darwin 1
 128 #elif defined(__APPLE__) && defined(__x86_64__)
 129 #  define PLAT_amd64_darwin 1
 130 #elif defined(__MINGW32__) || defined(__CYGWIN32__) || defined(_WIN32) && defined(_M_IX86)
 131 #  define PLAT_x86_win32 1
 132 #elif defined(__linux__) && defined(__i386__)
 133 #  define PLAT_x86_linux 1
 134 #elif defined(__linux__) && defined(__x86_64__)
 135 #  define PLAT_amd64_linux 1
 136 #elif defined(__linux__) && defined(__powerpc__) && !defined(__powerpc64__)
 137 #  define PLAT_ppc32_linux 1
 138 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__)
 139 #  define PLAT_ppc64_linux 1
 140 #elif defined(__linux__) && defined(__arm__)
 141 #  define PLAT_arm_linux 1
 142 #else
 143 /* If we're not compiling for our target platform, don't generate
 144    any inline asms.  */
 145 #  if !defined(NVALGRIND)
 146 #    define NVALGRIND 1
 147 #  endif
 148 #endif
 149
 150
 151 /* ------------------------------------------------------------------ */
 152 /* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS.  There is nothing */
 153 /* in here of use to end-users -- skip to the next section.           */
 154 /* ------------------------------------------------------------------ */
 155
 156 #if defined(NVALGRIND)
 157
 158 /* Define NVALGRIND to completely remove the Valgrind magic sequence
 159    from the compiled code (analogous to NDEBUG's effects on
 160    assert()) */
 161 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 162         _zzq_rlval, _zzq_default, _zzq_request,                   \
 163         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 164    {                                                              \
 165       (_zzq_rlval) = (_zzq_default);                              \
 166    }
 167
 168 #else  /* ! NVALGRIND */
 169
 170 /* The following defines the magic code sequences which the JITter
 171    spots and handles magically.  Don't look too closely at them as
 172    they will rot your brain.
 173
 174    The assembly code sequences for all architectures is in this one
 175    file.  This is because this file must be stand-alone, and we don't
 176    want to have multiple files.
 177
 178    For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default
 179    value gets put in the return slot, so that everything works when
 180    this is executed not under Valgrind.  Args are passed in a memory
 181    block, and so there's no intrinsic limit to the number that could
 182    be passed, but it's currently five.
 183
 184    The macro args are:
 185       _zzq_rlval    result lvalue
 186       _zzq_default  default value (result returned when running on real CPU)
 187       _zzq_request  request code
 188       _zzq_arg1..5  request params
 189
 190    The other two macros are used to support function wrapping, and are
 191    a lot simpler.  VALGRIND_GET_NR_CONTEXT returns the value of the
 192    guest's NRADDR pseudo-register and whatever other information is
 193    needed to safely run the call original from the wrapper: on
 194    ppc64-linux, the R2 value at the divert point is also needed.  This
 195    information is abstracted into a user-visible type, OrigFn.
 196
 197    VALGRIND_CALL_NOREDIR_* behaves the same as the following on the
 198    guest, but guarantees that the branch instruction will not be
 199    redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64:
 200    branch-and-link-to-r11.  VALGRIND_CALL_NOREDIR is just text, not a
 201    complete inline asm, since it needs to be combined with more magic
 202    inline asm stuff to be useful.
 203 */
 204
 205 /* ------------------------- x86-{linux,darwin} ---------------- */
 206
 207 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)  \
 208     ||  (defined(PLAT_x86_win32) && defined(__GNUC__))
 209
 210 typedef
 211    struct {
 212       unsigned int nraddr; /* where's the code? */
 213    }
 214    OrigFn;
 215
 216 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 217                      "roll $3,  %%edi ; roll $13, %%edi\n\t"      \
 218                      "roll $29, %%edi ; roll $19, %%edi\n\t"
 219
 220 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 221         _zzq_rlval, _zzq_default, _zzq_request,                   \
 222         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 223   { volatile unsigned int _zzq_args[6];                           \
 224     volatile unsigned int _zzq_result;                            \
 225     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 226     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 227     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 228     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 229     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 230     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 231     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 232                      /* %EDX = client_request ( %EAX ) */         \
 233                      "xchgl %%ebx,%%ebx"                          \
 234                      : "=d" (_zzq_result)                         \
 235                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 236                      : "cc", "memory"                             \
 237                     );                                            \
 238     _zzq_rlval = _zzq_result;                                     \
 239   }
 240
 241 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 242   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 243     volatile unsigned int __addr;                                 \
 244     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 245                      /* %EAX = guest_NRADDR */                    \
 246                      "xchgl %%ecx,%%ecx"                          \
 247                      : "=a" (__addr)                              \
 248                      :                                            \
 249                      : "cc", "memory"                             \
 250                     );                                            \
 251     _zzq_orig->nraddr = __addr;                                   \
 252   }
 253
 254 #define VALGRIND_CALL_NOREDIR_EAX                                 \
 255                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 256                      /* call-noredir *%EAX */                     \
 257                      "xchgl %%edx,%%edx\n\t"
 258 #endif /* PLAT_x86_linux || PLAT_x86_darwin || (PLAT_x86_win32 && __GNUC__) */
 259
 260 /* ------------------------- x86-Win32 ------------------------- */
 261
 262 #if defined(PLAT_x86_win32) && !defined(__GNUC__)
 263
 264 typedef
 265    struct {
 266       unsigned int nraddr; /* where's the code? */
 267    }
 268    OrigFn;
 269
 270 #if defined(_MSC_VER)
 271
 272 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 273                      __asm rol edi, 3  __asm rol edi, 13          \
 274                      __asm rol edi, 29 __asm rol edi, 19
 275
 276 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 277         _zzq_rlval, _zzq_default, _zzq_request,                   \
 278         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 279   { volatile uintptr_t _zzq_args[6];                              \
 280     volatile unsigned int _zzq_result;                            \
 281     _zzq_args[0] = (uintptr_t)(_zzq_request);                     \
 282     _zzq_args[1] = (uintptr_t)(_zzq_arg1);                        \
 283     _zzq_args[2] = (uintptr_t)(_zzq_arg2);                        \
 284     _zzq_args[3] = (uintptr_t)(_zzq_arg3);                        \
 285     _zzq_args[4] = (uintptr_t)(_zzq_arg4);                        \
 286     _zzq_args[5] = (uintptr_t)(_zzq_arg5);                        \
 287     __asm { __asm lea eax, _zzq_args __asm mov edx, _zzq_default  \
 288             __SPECIAL_INSTRUCTION_PREAMBLE                        \
 289             /* %EDX = client_request ( %EAX ) */                  \
 290             __asm xchg ebx,ebx                                    \
 291             __asm mov _zzq_result, edx                            \
 292     }                                                             \
 293     _zzq_rlval = _zzq_result;                                     \
 294   }
 295
 296 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 297   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 298     volatile unsigned int __addr;                                 \
 299     __asm { __SPECIAL_INSTRUCTION_PREAMBLE                        \
 300             /* %EAX = guest_NRADDR */                             \
 301             __asm xchg ecx,ecx                                    \
 302             __asm mov __addr, eax                                 \
 303     }                                                             \
 304     _zzq_orig->nraddr = __addr;                                   \
 305   }
 306
 307 #define VALGRIND_CALL_NOREDIR_EAX ERROR
 308
 309 #else
 310 #error Unsupported compiler.
 311 #endif
 312
 313 #endif /* PLAT_x86_win32 */
 314
 315 /* ------------------------ amd64-{linux,darwin} --------------- */
 316
 317 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin)
 318
 319 typedef
 320    struct {
 321       unsigned long long int nraddr; /* where's the code? */
 322    }
 323    OrigFn;
 324
 325 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 326                      "rolq $3,  %%rdi ; rolq $13, %%rdi\n\t"      \
 327                      "rolq $61, %%rdi ; rolq $51, %%rdi\n\t"
 328
 329 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 330         _zzq_rlval, _zzq_default, _zzq_request,                   \
 331         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 332   { volatile unsigned long long int _zzq_args[6];                 \
 333     volatile unsigned long long int _zzq_result;                  \
 334     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 335     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 336     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 337     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 338     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 339     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 340     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 341                      /* %RDX = client_request ( %RAX ) */         \
 342                      "xchgq %%rbx,%%rbx"                          \
 343                      : "=d" (_zzq_result)                         \
 344                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 345                      : "cc", "memory"                             \
 346                     );                                            \
 347     _zzq_rlval = _zzq_result;                                     \
 348   }
 349
 350 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 351   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 352     volatile unsigned long long int __addr;                       \
 353     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 354                      /* %RAX = guest_NRADDR */                    \
 355                      "xchgq %%rcx,%%rcx"                          \
 356                      : "=a" (__addr)                              \
 357                      :                                            \
 358                      : "cc", "memory"                             \
 359                     );                                            \
 360     _zzq_orig->nraddr = __addr;                                   \
 361   }
 362
 363 #define VALGRIND_CALL_NOREDIR_RAX                                 \
 364                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 365                      /* call-noredir *%RAX */                     \
 366                      "xchgq %%rdx,%%rdx\n\t"
 367 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
 368
 369 /* ------------------------ ppc32-linux ------------------------ */
 370
 371 #if defined(PLAT_ppc32_linux)
 372
 373 typedef
 374    struct {
 375       unsigned int nraddr; /* where's the code? */
 376    }
 377    OrigFn;
 378
 379 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 380                      "rlwinm 0,0,3,0,0  ; rlwinm 0,0,13,0,0\n\t"  \
 381                      "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
 382
 383 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 384         _zzq_rlval, _zzq_default, _zzq_request,                   \
 385         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 386                                                                   \
 387   {          unsigned int  _zzq_args[6];                          \
 388              unsigned int  _zzq_result;                           \
 389              unsigned int* _zzq_ptr;                              \
 390     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 391     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 392     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 393     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 394     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 395     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 396     _zzq_ptr = _zzq_args;                                         \
 397     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
 398                      "mr 4,%2\n\t" /*ptr*/                        \
 399                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 400                      /* %R3 = client_request ( %R4 ) */           \
 401                      "or 1,1,1\n\t"                               \
 402                      "mr %0,3"     /*result*/                     \
 403                      : "=b" (_zzq_result)                         \
 404                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
 405                      : "cc", "memory", "r3", "r4");               \
 406     _zzq_rlval = _zzq_result;                                     \
 407   }
 408
 409 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 410   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 411     unsigned int __addr;                                          \
 412     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 413                      /* %R3 = guest_NRADDR */                     \
 414                      "or 2,2,2\n\t"                               \
 415                      "mr %0,3"                                    \
 416                      : "=b" (__addr)                              \
 417                      :                                            \
 418                      : "cc", "memory", "r3"                       \
 419                     );                                            \
 420     _zzq_orig->nraddr = __addr;                                   \
 421   }
 422
 423 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 424                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 425                      /* branch-and-link-to-noredir *%R11 */       \
 426                      "or 3,3,3\n\t"
 427 #endif /* PLAT_ppc32_linux */
 428
 429 /* ------------------------ ppc64-linux ------------------------ */
 430
 431 #if defined(PLAT_ppc64_linux)
 432
 433 typedef
 434    struct {
 435       unsigned long long int nraddr; /* where's the code? */
 436       unsigned long long int r2;  /* what tocptr do we need? */
 437    }
 438    OrigFn;
 439
 440 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 441                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
 442                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
 443
 444 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 445         _zzq_rlval, _zzq_default, _zzq_request,                   \
 446         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 447                                                                   \
 448   {          unsigned long long int  _zzq_args[6];                \
 449     register unsigned long long int  _zzq_result __asm__("r3");   \
 450     register unsigned long long int* _zzq_ptr __asm__("r4");      \
 451     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 452     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 453     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 454     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 455     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 456     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 457     _zzq_ptr = _zzq_args;                                         \
 458     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 459                      /* %R3 = client_request ( %R4 ) */           \
 460                      "or 1,1,1"                                   \
 461                      : "=r" (_zzq_result)                         \
 462                      : "0" (_zzq_default), "r" (_zzq_ptr)         \
 463                      : "cc", "memory");                           \
 464     _zzq_rlval = _zzq_result;                                     \
 465   }
 466
 467 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 468   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 469     register unsigned long long int __addr __asm__("r3");         \
 470     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 471                      /* %R3 = guest_NRADDR */                     \
 472                      "or 2,2,2"                                   \
 473                      : "=r" (__addr)                              \
 474                      :                                            \
 475                      : "cc", "memory"                             \
 476                     );                                            \
 477     _zzq_orig->nraddr = __addr;                                   \
 478     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 479                      /* %R3 = guest_NRADDR_GPR2 */                \
 480                      "or 4,4,4"                                   \
 481                      : "=r" (__addr)                              \
 482                      :                                            \
 483                      : "cc", "memory"                             \
 484                     );                                            \
 485     _zzq_orig->r2 = __addr;                                       \
 486   }
 487
 488 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 489                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 490                      /* branch-and-link-to-noredir *%R11 */       \
 491                      "or 3,3,3\n\t"
 492
 493 #endif /* PLAT_ppc64_linux */
 494
 495 /* ------------------------- arm-linux ------------------------- */
 496
 497 #if defined(PLAT_arm_linux)
 498
 499 typedef
 500    struct {
 501       unsigned int nraddr; /* where's the code? */
 502    }
 503    OrigFn;
 504
 505 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 506             "mov r12, r12, ror #3  ; mov r12, r12, ror #13 \n\t"  \
 507             "mov r12, r12, ror #29 ; mov r12, r12, ror #19 \n\t"
 508
 509 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 510         _zzq_rlval, _zzq_default, _zzq_request,                   \
 511         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 512                                                                   \
 513   { volatile unsigned int  _zzq_args[6];                          \
 514     volatile unsigned int  _zzq_result;                           \
 515     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 516     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 517     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 518     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 519     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 520     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 521     __asm__ volatile("mov r3, %1\n\t" /*default*/                 \
 522                      "mov r4, %2\n\t" /*ptr*/                     \
 523                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 524                      /* R3 = client_request ( R4 ) */             \
 525                      "orr r10, r10, r10\n\t"                      \
 526                      "mov %0, r3"     /*result*/                  \
 527                      : "=r" (_zzq_result)                         \
 528                      : "r" (_zzq_default), "r" (&_zzq_args[0])    \
 529                      : "cc","memory", "r3", "r4");                \
 530     _zzq_rlval = _zzq_result;                                     \
 531   }
 532
 533 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 534   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 535     unsigned int __addr;                                          \
 536     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 537                      /* R3 = guest_NRADDR */                      \
 538                      "orr r11, r11, r11\n\t"                      \
 539                      "mov %0, r3"                                 \
 540                      : "=r" (__addr)                              \
 541                      :                                            \
 542                      : "cc", "memory", "r3"                       \
 543                     );                                            \
 544     _zzq_orig->nraddr = __addr;                                   \
 545   }
 546
 547 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                    \
 548                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 549                      /* branch-and-link-to-noredir *%R4 */        \
 550                      "orr r12, r12, r12\n\t"
 551
 552 #endif /* PLAT_arm_linux */
 553
 554 /* ------------------------ ppc32-aix5 ------------------------- */
 555
 556 #if defined(PLAT_ppc32_aix5)
 557
 558 typedef
 559    struct {
 560       unsigned int nraddr; /* where's the code? */
 561       unsigned int r2;  /* what tocptr do we need? */
 562    }
 563    OrigFn;
 564
 565 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 566                      "rlwinm 0,0,3,0,0  ; rlwinm 0,0,13,0,0\n\t"  \
 567                      "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
 568
 569 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 570         _zzq_rlval, _zzq_default, _zzq_request,                   \
 571         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 572                                                                   \
 573   {          unsigned int  _zzq_args[7];                          \
 574     register unsigned int  _zzq_result;                           \
 575     register unsigned int* _zzq_ptr;                              \
 576     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 577     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 578     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 579     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 580     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 581     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 582     _zzq_args[6] = (unsigned int)(_zzq_default);                  \
 583     _zzq_ptr = _zzq_args;                                         \
 584     __asm__ volatile("mr 4,%1\n\t"                                \
 585                      "lwz 3, 24(4)\n\t"                           \
 586                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 587                      /* %R3 = client_request ( %R4 ) */           \
 588                      "or 1,1,1\n\t"                               \
 589                      "mr %0,3"                                    \
 590                      : "=b" (_zzq_result)                         \
 591                      : "b" (_zzq_ptr)                             \
 592                      : "r3", "r4", "cc", "memory");               \
 593     _zzq_rlval = _zzq_result;                                     \
 594   }
 595
 596 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 597   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 598     register unsigned int __addr;                                 \
 599     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 600                      /* %R3 = guest_NRADDR */                     \
 601                      "or 2,2,2\n\t"                               \
 602                      "mr %0,3"                                    \
 603                      : "=b" (__addr)                              \
 604                      :                                            \
 605                      : "r3", "cc", "memory"                       \
 606                     );                                            \
 607     _zzq_orig->nraddr = __addr;                                   \
 608     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 609                      /* %R3 = guest_NRADDR_GPR2 */                \
 610                      "or 4,4,4\n\t"                               \
 611                      "mr %0,3"                                    \
 612                      : "=b" (__addr)                              \
 613                      :                                            \
 614                      : "r3", "cc", "memory"                       \
 615                     );                                            \
 616     _zzq_orig->r2 = __addr;                                       \
 617   }
 618
 619 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 620                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 621                      /* branch-and-link-to-noredir *%R11 */       \
 622                      "or 3,3,3\n\t"
 623
 624 #endif /* PLAT_ppc32_aix5 */
 625
 626 /* ------------------------ ppc64-aix5 ------------------------- */
 627
 628 #if defined(PLAT_ppc64_aix5)
 629
 630 typedef
 631    struct {
 632       unsigned long long int nraddr; /* where's the code? */
 633       unsigned long long int r2;  /* what tocptr do we need? */
 634    }
 635    OrigFn;
 636
 637 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 638                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
 639                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
 640
 641 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 642         _zzq_rlval, _zzq_default, _zzq_request,                   \
 643         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 644                                                                   \
 645   {          unsigned long long int  _zzq_args[7];                \
 646     register unsigned long long int  _zzq_result;                 \
 647     register unsigned long long int* _zzq_ptr;                    \
 648     _zzq_args[0] = (unsigned int long long)(_zzq_request);        \
 649     _zzq_args[1] = (unsigned int long long)(_zzq_arg1);           \
 650     _zzq_args[2] = (unsigned int long long)(_zzq_arg2);           \
 651     _zzq_args[3] = (unsigned int long long)(_zzq_arg3);           \
 652     _zzq_args[4] = (unsigned int long long)(_zzq_arg4);           \
 653     _zzq_args[5] = (unsigned int long long)(_zzq_arg5);           \
 654     _zzq_args[6] = (unsigned int long long)(_zzq_default);        \
 655     _zzq_ptr = _zzq_args;                                         \
 656     __asm__ volatile("mr 4,%1\n\t"                                \
 657                      "ld 3, 48(4)\n\t"                            \
 658                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 659                      /* %R3 = client_request ( %R4 ) */           \
 660                      "or 1,1,1\n\t"                               \
 661                      "mr %0,3"                                    \
 662                      : "=b" (_zzq_result)                         \
 663                      : "b" (_zzq_ptr)                             \
 664                      : "r3", "r4", "cc", "memory");               \
 665     _zzq_rlval = _zzq_result;                                     \
 666   }
 667
 668 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 669   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 670     register unsigned long long int __addr;                       \
 671     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 672                      /* %R3 = guest_NRADDR */                     \
 673                      "or 2,2,2\n\t"                               \
 674                      "mr %0,3"                                    \
 675                      : "=b" (__addr)                              \
 676                      :                                            \
 677                      : "r3", "cc", "memory"                       \
 678                     );                                            \
 679     _zzq_orig->nraddr = __addr;                                   \
 680     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 681                      /* %R3 = guest_NRADDR_GPR2 */                \
 682                      "or 4,4,4\n\t"                               \
 683                      "mr %0,3"                                    \
 684                      : "=b" (__addr)                              \
 685                      :                                            \
 686                      : "r3", "cc", "memory"                       \
 687                     );                                            \
 688     _zzq_orig->r2 = __addr;                                       \
 689   }
 690
 691 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 692                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 693                      /* branch-and-link-to-noredir *%R11 */       \
 694                      "or 3,3,3\n\t"
 695
 696 #endif /* PLAT_ppc64_aix5 */
 697
 698 /* Insert assembly code for other platforms here... */
 699
 700 #endif /* NVALGRIND */
 701
 702
 703 /* ------------------------------------------------------------------ */
 704 /* PLATFORM SPECIFICS for FUNCTION WRAPPING.  This is all very        */
 705 /* ugly.  It's the least-worst tradeoff I can think of.               */
 706 /* ------------------------------------------------------------------ */
 707
 708 /* This section defines magic (a.k.a appalling-hack) macros for doing
 709    guaranteed-no-redirection macros, so as to get from function
 710    wrappers to the functions they are wrapping.  The whole point is to
 711    construct standard call sequences, but to do the call itself with a
 712    special no-redirect call pseudo-instruction that the JIT
 713    understands and handles specially.  This section is long and
 714    repetitious, and I can't see a way to make it shorter.
 715
 716    The naming scheme is as follows:
 717
 718       CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc}
 719
 720    'W' stands for "word" and 'v' for "void".  Hence there are
 721    different macros for calling arity 0, 1, 2, 3, 4, etc, functions,
 722    and for each, the possibility of returning a word-typed result, or
 723    no result.
 724 */
 725
 726 /* Use these to write the name of your wrapper.  NOTE: duplicates
 727    VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h. */
 728
 729 /* Use an extra level of macroisation so as to ensure the soname/fnname
 730    args are fully macro-expanded before pasting them together. */
 731 #define VG_CONCAT4(_aa,_bb,_cc,_dd) _aa##_bb##_cc##_dd
 732
 733 #define I_WRAP_SONAME_FNNAME_ZU(soname,fnname)                    \
 734    VG_CONCAT4(_vgwZU_,soname,_,fnname)
 735
 736 #define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname)                    \
 737    VG_CONCAT4(_vgwZZ_,soname,_,fnname)
 738
 739 /* Use this macro from within a wrapper function to collect the
 740    context (address and possibly other info) of the original function.
 741    Once you have that you can then use it in one of the CALL_FN_
 742    macros.  The type of the argument _lval is OrigFn. */
 743 #define VALGRIND_GET_ORIG_FN(_lval)  VALGRIND_GET_NR_CONTEXT(_lval)
 744
 745 /* Derivatives of the main macros below, for calling functions
 746    returning void. */
 747
 748 #define CALL_FN_v_v(fnptr)                                        \
 749    do { volatile unsigned long _junk;                             \
 750         CALL_FN_W_v(_junk,fnptr); } while (0)
 751
 752 #define CALL_FN_v_W(fnptr, arg1)                                  \
 753    do { volatile unsigned long _junk;                             \
 754         CALL_FN_W_W(_junk,fnptr,arg1); } while (0)
 755
 756 #define CALL_FN_v_WW(fnptr, arg1,arg2)                            \
 757    do { volatile unsigned long _junk;                             \
 758         CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0)
 759
 760 #define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3)                      \
 761    do { volatile unsigned long _junk;                             \
 762         CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0)
 763
 764 #define CALL_FN_v_WWWW(fnptr, arg1,arg2,arg3,arg4)                \
 765    do { volatile unsigned long _junk;                             \
 766         CALL_FN_W_WWWW(_junk,fnptr,arg1,arg2,arg3,arg4); } while (0)
 767
 768 #define CALL_FN_v_5W(fnptr, arg1,arg2,arg3,arg4,arg5)             \
 769    do { volatile unsigned long _junk;                             \
 770         CALL_FN_W_5W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5); } while (0)
 771
 772 #define CALL_FN_v_6W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6)        \
 773    do { volatile unsigned long _junk;                             \
 774         CALL_FN_W_6W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6); } while (0)
 775
 776 #define CALL_FN_v_7W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6,arg7)   \
 777    do { volatile unsigned long _junk;                             \
 778         CALL_FN_W_7W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6,arg7); } while (0)
 779
 780 /* ------------------------- x86-{linux,darwin} ---------------- */
 781
 782 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)
 783
 784 /* These regs are trashed by the hidden call.  No need to mention eax
 785    as gcc can already see that, plus causes gcc to bomb. */
 786 #define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx"
 787
 788 /* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned
 789    long) == 4. */
 790
 791 #define CALL_FN_W_v(lval, orig)                                   \
 792    do {                                                           \
 793       volatile OrigFn        _orig = (orig);                      \
 794       volatile unsigned long _argvec[1];                          \
 795       volatile unsigned long _res;                                \
 796       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 797       __asm__ volatile(                                           \
 798          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 799          VALGRIND_CALL_NOREDIR_EAX                                \
 800          : /*out*/   "=a" (_res)                                  \
 801          : /*in*/    "a" (&_argvec[0])                            \
 802          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 803       );                                                          \
 804       lval = (__typeof__(lval)) _res;                             \
 805    } while (0)
 806
 807 #define CALL_FN_W_W(lval, orig, arg1)                             \
 808    do {                                                           \
 809       volatile OrigFn        _orig = (orig);                      \
 810       volatile unsigned long _argvec[2];                          \
 811       volatile unsigned long _res;                                \
 812       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 813       _argvec[1] = (unsigned long)(arg1);                         \
 814       __asm__ volatile(                                           \
 815          "subl $12, %%esp\n\t"                                    \
 816          "pushl 4(%%eax)\n\t"                                     \
 817          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 818          VALGRIND_CALL_NOREDIR_EAX                                \
 819          "addl $16, %%esp\n"                                      \
 820          : /*out*/   "=a" (_res)                                  \
 821          : /*in*/    "a" (&_argvec[0])                            \
 822          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 823       );                                                          \
 824       lval = (__typeof__(lval)) _res;                             \
 825    } while (0)
 826
 827 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
 828    do {                                                           \
 829       volatile OrigFn        _orig = (orig);                      \
 830       volatile unsigned long _argvec[3];                          \
 831       volatile unsigned long _res;                                \
 832       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 833       _argvec[1] = (unsigned long)(arg1);                         \
 834       _argvec[2] = (unsigned long)(arg2);                         \
 835       __asm__ volatile(                                           \
 836          "subl $8, %%esp\n\t"                                     \
 837          "pushl 8(%%eax)\n\t"                                     \
 838          "pushl 4(%%eax)\n\t"                                     \
 839          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 840          VALGRIND_CALL_NOREDIR_EAX                                \
 841          "addl $16, %%esp\n"                                      \
 842          : /*out*/   "=a" (_res)                                  \
 843          : /*in*/    "a" (&_argvec[0])                            \
 844          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 845       );                                                          \
 846       lval = (__typeof__(lval)) _res;                             \
 847    } while (0)
 848
 849 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
 850    do {                                                           \
 851       volatile OrigFn        _orig = (orig);                      \
 852       volatile unsigned long _argvec[4];                          \
 853       volatile unsigned long _res;                                \
 854       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 855       _argvec[1] = (unsigned long)(arg1);                         \
 856       _argvec[2] = (unsigned long)(arg2);                         \
 857       _argvec[3] = (unsigned long)(arg3);                         \
 858       __asm__ volatile(                                           \
 859          "subl $4, %%esp\n\t"                                     \
 860          "pushl 12(%%eax)\n\t"                                    \
 861          "pushl 8(%%eax)\n\t"                                     \
 862          "pushl 4(%%eax)\n\t"                                     \
 863          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 864          VALGRIND_CALL_NOREDIR_EAX                                \
 865          "addl $16, %%esp\n"                                      \
 866          : /*out*/   "=a" (_res)                                  \
 867          : /*in*/    "a" (&_argvec[0])                            \
 868          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 869       );                                                          \
 870       lval = (__typeof__(lval)) _res;                             \
 871    } while (0)
 872
 873 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
 874    do {                                                           \
 875       volatile OrigFn        _orig = (orig);                      \
 876       volatile unsigned long _argvec[5];                          \
 877       volatile unsigned long _res;                                \
 878       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 879       _argvec[1] = (unsigned long)(arg1);                         \
 880       _argvec[2] = (unsigned long)(arg2);                         \
 881       _argvec[3] = (unsigned long)(arg3);                         \
 882       _argvec[4] = (unsigned long)(arg4);                         \
 883       __asm__ volatile(                                           \
 884          "pushl 16(%%eax)\n\t"                                    \
 885          "pushl 12(%%eax)\n\t"                                    \
 886          "pushl 8(%%eax)\n\t"                                     \
 887          "pushl 4(%%eax)\n\t"                                     \
 888          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 889          VALGRIND_CALL_NOREDIR_EAX                                \
 890          "addl $16, %%esp\n"                                      \
 891          : /*out*/   "=a" (_res)                                  \
 892          : /*in*/    "a" (&_argvec[0])                            \
 893          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 894       );                                                          \
 895       lval = (__typeof__(lval)) _res;                             \
 896    } while (0)
 897
 898 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
 899    do {                                                           \
 900       volatile OrigFn        _orig = (orig);                      \
 901       volatile unsigned long _argvec[6];                          \
 902       volatile unsigned long _res;                                \
 903       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 904       _argvec[1] = (unsigned long)(arg1);                         \
 905       _argvec[2] = (unsigned long)(arg2);                         \
 906       _argvec[3] = (unsigned long)(arg3);                         \
 907       _argvec[4] = (unsigned long)(arg4);                         \
 908       _argvec[5] = (unsigned long)(arg5);                         \
 909       __asm__ volatile(                                           \
 910          "subl $12, %%esp\n\t"                                    \
 911          "pushl 20(%%eax)\n\t"                                    \
 912          "pushl 16(%%eax)\n\t"                                    \
 913          "pushl 12(%%eax)\n\t"                                    \
 914          "pushl 8(%%eax)\n\t"                                     \
 915          "pushl 4(%%eax)\n\t"                                     \
 916          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 917          VALGRIND_CALL_NOREDIR_EAX                                \
 918          "addl $32, %%esp\n"                                      \
 919          : /*out*/   "=a" (_res)                                  \
 920          : /*in*/    "a" (&_argvec[0])                            \
 921          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 922       );                                                          \
 923       lval = (__typeof__(lval)) _res;                             \
 924    } while (0)
 925
 926 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
 927    do {                                                           \
 928       volatile OrigFn        _orig = (orig);                      \
 929       volatile unsigned long _argvec[7];                          \
 930       volatile unsigned long _res;                                \
 931       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 932       _argvec[1] = (unsigned long)(arg1);                         \
 933       _argvec[2] = (unsigned long)(arg2);                         \
 934       _argvec[3] = (unsigned long)(arg3);                         \
 935       _argvec[4] = (unsigned long)(arg4);                         \
 936       _argvec[5] = (unsigned long)(arg5);                         \
 937       _argvec[6] = (unsigned long)(arg6);                         \
 938       __asm__ volatile(                                           \
 939          "subl $8, %%esp\n\t"                                     \
 940          "pushl 24(%%eax)\n\t"                                    \
 941          "pushl 20(%%eax)\n\t"                                    \
 942          "pushl 16(%%eax)\n\t"                                    \
 943          "pushl 12(%%eax)\n\t"                                    \
 944          "pushl 8(%%eax)\n\t"                                     \
 945          "pushl 4(%%eax)\n\t"                                     \
 946          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 947          VALGRIND_CALL_NOREDIR_EAX                                \
 948          "addl $32, %%esp\n"                                      \
 949          : /*out*/   "=a" (_res)                                  \
 950          : /*in*/    "a" (&_argvec[0])                            \
 951          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 952       );                                                          \
 953       lval = (__typeof__(lval)) _res;                             \
 954    } while (0)
 955
 956 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
 957                                  arg7)                            \
 958    do {                                                           \
 959       volatile OrigFn        _orig = (orig);                      \
 960       volatile unsigned long _argvec[8];                          \
 961       volatile unsigned long _res;                                \
 962       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 963       _argvec[1] = (unsigned long)(arg1);                         \
 964       _argvec[2] = (unsigned long)(arg2);                         \
 965       _argvec[3] = (unsigned long)(arg3);                         \
 966       _argvec[4] = (unsigned long)(arg4);                         \
 967       _argvec[5] = (unsigned long)(arg5);                         \
 968       _argvec[6] = (unsigned long)(arg6);                         \
 969       _argvec[7] = (unsigned long)(arg7);                         \
 970       __asm__ volatile(                                           \
 971          "subl $4, %%esp\n\t"                                     \
 972          "pushl 28(%%eax)\n\t"                                    \
 973          "pushl 24(%%eax)\n\t"                                    \
 974          "pushl 20(%%eax)\n\t"                                    \
 975          "pushl 16(%%eax)\n\t"                                    \
 976          "pushl 12(%%eax)\n\t"                                    \
 977          "pushl 8(%%eax)\n\t"                                     \
 978          "pushl 4(%%eax)\n\t"                                     \
 979          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 980          VALGRIND_CALL_NOREDIR_EAX                                \
 981          "addl $32, %%esp\n"                                      \
 982          : /*out*/   "=a" (_res)                                  \
 983          : /*in*/    "a" (&_argvec[0])                            \
 984          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 985       );                                                          \
 986       lval = (__typeof__(lval)) _res;                             \
 987    } while (0)
 988
 989 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
 990                                  arg7,arg8)                       \
 991    do {                                                           \
 992       volatile OrigFn        _orig = (orig);                      \
 993       volatile unsigned long _argvec[9];                          \
 994       volatile unsigned long _res;                                \
 995       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 996       _argvec[1] = (unsigned long)(arg1);                         \
 997       _argvec[2] = (unsigned long)(arg2);                         \
 998       _argvec[3] = (unsigned long)(arg3);                         \
 999       _argvec[4] = (unsigned long)(arg4);                         \
1000       _argvec[5] = (unsigned long)(arg5);                         \
1001       _argvec[6] = (unsigned long)(arg6);                         \
1002       _argvec[7] = (unsigned long)(arg7);                         \
1003       _argvec[8] = (unsigned long)(arg8);                         \
1004       __asm__ volatile(                                           \
1005          "pushl 32(%%eax)\n\t"                                    \
1006          "pushl 28(%%eax)\n\t"                                    \
1007          "pushl 24(%%eax)\n\t"                                    \
1008          "pushl 20(%%eax)\n\t"                                    \
1009          "pushl 16(%%eax)\n\t"                                    \
1010          "pushl 12(%%eax)\n\t"                                    \
1011          "pushl 8(%%eax)\n\t"                                     \
1012          "pushl 4(%%eax)\n\t"                                     \
1013          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1014          VALGRIND_CALL_NOREDIR_EAX                                \
1015          "addl $32, %%esp\n"                                      \
1016          : /*out*/   "=a" (_res)                                  \
1017          : /*in*/    "a" (&_argvec[0])                            \
1018          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1019       );                                                          \
1020       lval = (__typeof__(lval)) _res;                             \
1021    } while (0)
1022
1023 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1024                                  arg7,arg8,arg9)                  \
1025    do {                                                           \
1026       volatile OrigFn        _orig = (orig);                      \
1027       volatile unsigned long _argvec[10];                         \
1028       volatile unsigned long _res;                                \
1029       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1030       _argvec[1] = (unsigned long)(arg1);                         \
1031       _argvec[2] = (unsigned long)(arg2);                         \
1032       _argvec[3] = (unsigned long)(arg3);                         \
1033       _argvec[4] = (unsigned long)(arg4);                         \
1034       _argvec[5] = (unsigned long)(arg5);                         \
1035       _argvec[6] = (unsigned long)(arg6);                         \
1036       _argvec[7] = (unsigned long)(arg7);                         \
1037       _argvec[8] = (unsigned long)(arg8);                         \
1038       _argvec[9] = (unsigned long)(arg9);                         \
1039       __asm__ volatile(                                           \
1040          "subl $12, %%esp\n\t"                                    \
1041          "pushl 36(%%eax)\n\t"                                    \
1042          "pushl 32(%%eax)\n\t"                                    \
1043          "pushl 28(%%eax)\n\t"                                    \
1044          "pushl 24(%%eax)\n\t"                                    \
1045          "pushl 20(%%eax)\n\t"                                    \
1046          "pushl 16(%%eax)\n\t"                                    \
1047          "pushl 12(%%eax)\n\t"                                    \
1048          "pushl 8(%%eax)\n\t"                                     \
1049          "pushl 4(%%eax)\n\t"                                     \
1050          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1051          VALGRIND_CALL_NOREDIR_EAX                                \
1052          "addl $48, %%esp\n"                                      \
1053          : /*out*/   "=a" (_res)                                  \
1054          : /*in*/    "a" (&_argvec[0])                            \
1055          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1056       );                                                          \
1057       lval = (__typeof__(lval)) _res;                             \
1058    } while (0)
1059
1060 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1061                                   arg7,arg8,arg9,arg10)           \
1062    do {                                                           \
1063       volatile OrigFn        _orig = (orig);                      \
1064       volatile unsigned long _argvec[11];                         \
1065       volatile unsigned long _res;                                \
1066       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1067       _argvec[1] = (unsigned long)(arg1);                         \
1068       _argvec[2] = (unsigned long)(arg2);                         \
1069       _argvec[3] = (unsigned long)(arg3);                         \
1070       _argvec[4] = (unsigned long)(arg4);                         \
1071       _argvec[5] = (unsigned long)(arg5);                         \
1072       _argvec[6] = (unsigned long)(arg6);                         \
1073       _argvec[7] = (unsigned long)(arg7);                         \
1074       _argvec[8] = (unsigned long)(arg8);                         \
1075       _argvec[9] = (unsigned long)(arg9);                         \
1076       _argvec[10] = (unsigned long)(arg10);                       \
1077       __asm__ volatile(                                           \
1078          "subl $8, %%esp\n\t"                                     \
1079          "pushl 40(%%eax)\n\t"                                    \
1080          "pushl 36(%%eax)\n\t"                                    \
1081          "pushl 32(%%eax)\n\t"                                    \
1082          "pushl 28(%%eax)\n\t"                                    \
1083          "pushl 24(%%eax)\n\t"                                    \
1084          "pushl 20(%%eax)\n\t"                                    \
1085          "pushl 16(%%eax)\n\t"                                    \
1086          "pushl 12(%%eax)\n\t"                                    \
1087          "pushl 8(%%eax)\n\t"                                     \
1088          "pushl 4(%%eax)\n\t"                                     \
1089          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1090          VALGRIND_CALL_NOREDIR_EAX                                \
1091          "addl $48, %%esp\n"                                      \
1092          : /*out*/   "=a" (_res)                                  \
1093          : /*in*/    "a" (&_argvec[0])                            \
1094          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1095       );                                                          \
1096       lval = (__typeof__(lval)) _res;                             \
1097    } while (0)
1098
1099 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1100                                   arg6,arg7,arg8,arg9,arg10,      \
1101                                   arg11)                          \
1102    do {                                                           \
1103       volatile OrigFn        _orig = (orig);                      \
1104       volatile unsigned long _argvec[12];                         \
1105       volatile unsigned long _res;                                \
1106       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1107       _argvec[1] = (unsigned long)(arg1);                         \
1108       _argvec[2] = (unsigned long)(arg2);                         \
1109       _argvec[3] = (unsigned long)(arg3);                         \
1110       _argvec[4] = (unsigned long)(arg4);                         \
1111       _argvec[5] = (unsigned long)(arg5);                         \
1112       _argvec[6] = (unsigned long)(arg6);                         \
1113       _argvec[7] = (unsigned long)(arg7);                         \
1114       _argvec[8] = (unsigned long)(arg8);                         \
1115       _argvec[9] = (unsigned long)(arg9);                         \
1116       _argvec[10] = (unsigned long)(arg10);                       \
1117       _argvec[11] = (unsigned long)(arg11);                       \
1118       __asm__ volatile(                                           \
1119          "subl $4, %%esp\n\t"                                     \
1120          "pushl 44(%%eax)\n\t"                                    \
1121          "pushl 40(%%eax)\n\t"                                    \
1122          "pushl 36(%%eax)\n\t"                                    \
1123          "pushl 32(%%eax)\n\t"                                    \
1124          "pushl 28(%%eax)\n\t"                                    \
1125          "pushl 24(%%eax)\n\t"                                    \
1126          "pushl 20(%%eax)\n\t"                                    \
1127          "pushl 16(%%eax)\n\t"                                    \
1128          "pushl 12(%%eax)\n\t"                                    \
1129          "pushl 8(%%eax)\n\t"                                     \
1130          "pushl 4(%%eax)\n\t"                                     \
1131          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1132          VALGRIND_CALL_NOREDIR_EAX                                \
1133          "addl $48, %%esp\n"                                      \
1134          : /*out*/   "=a" (_res)                                  \
1135          : /*in*/    "a" (&_argvec[0])                            \
1136          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1137       );                                                          \
1138       lval = (__typeof__(lval)) _res;                             \
1139    } while (0)
1140
1141 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1142                                   arg6,arg7,arg8,arg9,arg10,      \
1143                                   arg11,arg12)                    \
1144    do {                                                           \
1145       volatile OrigFn        _orig = (orig);                      \
1146       volatile unsigned long _argvec[13];                         \
1147       volatile unsigned long _res;                                \
1148       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1149       _argvec[1] = (unsigned long)(arg1);                         \
1150       _argvec[2] = (unsigned long)(arg2);                         \
1151       _argvec[3] = (unsigned long)(arg3);                         \
1152       _argvec[4] = (unsigned long)(arg4);                         \
1153       _argvec[5] = (unsigned long)(arg5);                         \
1154       _argvec[6] = (unsigned long)(arg6);                         \
1155       _argvec[7] = (unsigned long)(arg7);                         \
1156       _argvec[8] = (unsigned long)(arg8);                         \
1157       _argvec[9] = (unsigned long)(arg9);                         \
1158       _argvec[10] = (unsigned long)(arg10);                       \
1159       _argvec[11] = (unsigned long)(arg11);                       \
1160       _argvec[12] = (unsigned long)(arg12);                       \
1161       __asm__ volatile(                                           \
1162          "pushl 48(%%eax)\n\t"                                    \
1163          "pushl 44(%%eax)\n\t"                                    \
1164          "pushl 40(%%eax)\n\t"                                    \
1165          "pushl 36(%%eax)\n\t"                                    \
1166          "pushl 32(%%eax)\n\t"                                    \
1167          "pushl 28(%%eax)\n\t"                                    \
1168          "pushl 24(%%eax)\n\t"                                    \
1169          "pushl 20(%%eax)\n\t"                                    \
1170          "pushl 16(%%eax)\n\t"                                    \
1171          "pushl 12(%%eax)\n\t"                                    \
1172          "pushl 8(%%eax)\n\t"                                     \
1173          "pushl 4(%%eax)\n\t"                                     \
1174          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1175          VALGRIND_CALL_NOREDIR_EAX                                \
1176          "addl $48, %%esp\n"                                      \
1177          : /*out*/   "=a" (_res)                                  \
1178          : /*in*/    "a" (&_argvec[0])                            \
1179          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1180       );                                                          \
1181       lval = (__typeof__(lval)) _res;                             \
1182    } while (0)
1183
1184 #endif /* PLAT_x86_linux || PLAT_x86_darwin */
1185
1186 /* ------------------------ amd64-{linux,darwin} --------------- */
1187
1188 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin)
1189
1190 /* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */
1191
1192 /* These regs are trashed by the hidden call. */
1193 #define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi",       \
1194                             "rdi", "r8", "r9", "r10", "r11"
1195
1196 /* This is all pretty complex.  It's so as to make stack unwinding
1197    work reliably.  See bug 243270.  The basic problem is the sub and
1198    add of 128 of %rsp in all of the following macros.  If gcc believes
1199    the CFA is in %rsp, then unwinding may fail, because what's at the
1200    CFA is not what gcc "expected" when it constructs the CFIs for the
1201    places where the macros are instantiated.
1202
1203    But we can't just add a CFI annotation to increase the CFA offset
1204    by 128, to match the sub of 128 from %rsp, because we don't know
1205    whether gcc has chosen %rsp as the CFA at that point, or whether it
1206    has chosen some other register (eg, %rbp).  In the latter case,
1207    adding a CFI annotation to change the CFA offset is simply wrong.
1208
1209    So the solution is to get hold of the CFA using
1210    __builtin_dwarf_cfa(), put it in a known register, and add a
1211    CFI annotation to say what the register is.  We choose %rbp for
1212    this (perhaps perversely), because:
1213
1214    (1) %rbp is already subject to unwinding.  If a new register was
1215        chosen then the unwinder would have to unwind it in all stack
1216        traces, which is expensive, and
1217
1218    (2) %rbp is already subject to precise exception updates in the
1219        JIT.  If a new register was chosen, we'd have to have precise
1220        exceptions for it too, which reduces performance of the
1221        generated code.
1222
1223    However .. one extra complication.  We can't just whack the result
1224    of __builtin_dwarf_cfa() into %rbp and then add %rbp to the
1225    list of trashed registers at the end of the inline assembly
1226    fragments; gcc won't allow %rbp to appear in that list.  Hence
1227    instead we need to stash %rbp in %r15 for the duration of the asm,
1228    and say that %r15 is trashed instead.  gcc seems happy to go with
1229    that.
1230
1231    Oh .. and this all needs to be conditionalised so that it is
1232    unchanged from before this commit, when compiled with older gccs
1233    that don't support __builtin_dwarf_cfa.  Furthermore, since
1234    this header file is freestanding, it has to be independent of
1235    config.h, and so the following conditionalisation cannot depend on
1236    configure time checks.
1237
1238    Although it's not clear from
1239    'defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)',
1240    this expression excludes Darwin.
1241    .cfi directives in Darwin assembly appear to be completely
1242    different and I haven't investigated how they work.
1243
1244    For even more entertainment value, note we have to use the
1245    completely undocumented __builtin_dwarf_cfa(), which appears to
1246    really compute the CFA, whereas __builtin_frame_address(0) claims
1247    to but actually doesn't.  See
1248    https://bugs.kde.org/show_bug.cgi?id=243270#c47
1249 */
1250 #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)
1251 #  define __FRAME_POINTER                                         \
1252       ,"r"(__builtin_dwarf_cfa())
1253 #  define VALGRIND_CFI_PROLOGUE                                   \
1254       "movq %%rbp, %%r15\n\t"                                     \
1255       "movq %2, %%rbp\n\t"                                        \
1256       ".cfi_remember_state\n\t"                                   \
1257       ".cfi_def_cfa rbp, 0\n\t"
1258 #  define VALGRIND_CFI_EPILOGUE                                   \
1259       "movq %%r15, %%rbp\n\t"                                     \
1260       ".cfi_restore_state\n\t"
1261 #else
1262 #  define __FRAME_POINTER
1263 #  define VALGRIND_CFI_PROLOGUE
1264 #  define VALGRIND_CFI_EPILOGUE
1265 #endif
1266
1267
1268 /* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned
1269    long) == 8. */
1270
1271 /* NB 9 Sept 07.  There is a nasty kludge here in all these CALL_FN_
1272    macros.  In order not to trash the stack redzone, we need to drop
1273    %rsp by 128 before the hidden call, and restore afterwards.  The
1274    nastyness is that it is only by luck that the stack still appears
1275    to be unwindable during the hidden call - since then the behaviour
1276    of any routine using this macro does not match what the CFI data
1277    says.  Sigh.
1278
1279    Why is this important?  Imagine that a wrapper has a stack
1280    allocated local, and passes to the hidden call, a pointer to it.
1281    Because gcc does not know about the hidden call, it may allocate
1282    that local in the redzone.  Unfortunately the hidden call may then
1283    trash it before it comes to use it.  So we must step clear of the
1284    redzone, for the duration of the hidden call, to make it safe.
1285
1286    Probably the same problem afflicts the other redzone-style ABIs too
1287    (ppc64-linux, ppc32-aix5, ppc64-aix5); but for those, the stack is
1288    self describing (none of this CFI nonsense) so at least messing
1289    with the stack pointer doesn't give a danger of non-unwindable
1290    stack. */
1291
1292 #define CALL_FN_W_v(lval, orig)                                   \
1293    do {                                                           \
1294       volatile OrigFn        _orig = (orig);                      \
1295       volatile unsigned long _argvec[1];                          \
1296       volatile unsigned long _res;                                \
1297       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1298       __asm__ volatile(                                           \
1299          VALGRIND_CFI_PROLOGUE                                    \
1300          "subq $128,%%rsp\n\t"                                    \
1301          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1302          VALGRIND_CALL_NOREDIR_RAX                                \
1303          "addq $128,%%rsp\n\t"                                    \
1304          VALGRIND_CFI_EPILOGUE                                    \
1305          : /*out*/   "=a" (_res)                                  \
1306          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1307          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1308       );                                                          \
1309       lval = (__typeof__(lval)) _res;                             \
1310    } while (0)
1311
1312 #define CALL_FN_W_W(lval, orig, arg1)                             \
1313    do {                                                           \
1314       volatile OrigFn        _orig = (orig);                      \
1315       volatile unsigned long _argvec[2];                          \
1316       volatile unsigned long _res;                                \
1317       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1318       _argvec[1] = (unsigned long)(arg1);                         \
1319       __asm__ volatile(                                           \
1320          VALGRIND_CFI_PROLOGUE                                    \
1321          "subq $128,%%rsp\n\t"                                    \
1322          "movq 8(%%rax), %%rdi\n\t"                               \
1323          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1324          VALGRIND_CALL_NOREDIR_RAX                                \
1325          "addq $128,%%rsp\n\t"                                    \
1326          VALGRIND_CFI_EPILOGUE                                    \
1327          : /*out*/   "=a" (_res)                                  \
1328          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1329          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1330       );                                                          \
1331       lval = (__typeof__(lval)) _res;                             \
1332    } while (0)
1333
1334 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1335    do {                                                           \
1336       volatile OrigFn        _orig = (orig);                      \
1337       volatile unsigned long _argvec[3];                          \
1338       volatile unsigned long _res;                                \
1339       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1340       _argvec[1] = (unsigned long)(arg1);                         \
1341       _argvec[2] = (unsigned long)(arg2);                         \
1342       __asm__ volatile(                                           \
1343          VALGRIND_CFI_PROLOGUE                                    \
1344          "subq $128,%%rsp\n\t"                                    \
1345          "movq 16(%%rax), %%rsi\n\t"                              \
1346          "movq 8(%%rax), %%rdi\n\t"                               \
1347          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1348          VALGRIND_CALL_NOREDIR_RAX                                \
1349          "addq $128,%%rsp\n\t"                                    \
1350          VALGRIND_CFI_EPILOGUE                                    \
1351          : /*out*/   "=a" (_res)                                  \
1352          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1353          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1354       );                                                          \
1355       lval = (__typeof__(lval)) _res;                             \
1356    } while (0)
1357
1358 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1359    do {                                                           \
1360       volatile OrigFn        _orig = (orig);                      \
1361       volatile unsigned long _argvec[4];                          \
1362       volatile unsigned long _res;                                \
1363       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1364       _argvec[1] = (unsigned long)(arg1);                         \
1365       _argvec[2] = (unsigned long)(arg2);                         \
1366       _argvec[3] = (unsigned long)(arg3);                         \
1367       __asm__ volatile(                                           \
1368          VALGRIND_CFI_PROLOGUE                                    \
1369          "subq $128,%%rsp\n\t"                                    \
1370          "movq 24(%%rax), %%rdx\n\t"                              \
1371          "movq 16(%%rax), %%rsi\n\t"                              \
1372          "movq 8(%%rax), %%rdi\n\t"                               \
1373          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1374          VALGRIND_CALL_NOREDIR_RAX                                \
1375          "addq $128,%%rsp\n\t"                                    \
1376          VALGRIND_CFI_EPILOGUE                                    \
1377          : /*out*/   "=a" (_res)                                  \
1378          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1379          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1380       );                                                          \
1381       lval = (__typeof__(lval)) _res;                             \
1382    } while (0)
1383
1384 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1385    do {                                                           \
1386       volatile OrigFn        _orig = (orig);                      \
1387       volatile unsigned long _argvec[5];                          \
1388       volatile unsigned long _res;                                \
1389       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1390       _argvec[1] = (unsigned long)(arg1);                         \
1391       _argvec[2] = (unsigned long)(arg2);                         \
1392       _argvec[3] = (unsigned long)(arg3);                         \
1393       _argvec[4] = (unsigned long)(arg4);                         \
1394       __asm__ volatile(                                           \
1395          VALGRIND_CFI_PROLOGUE                                    \
1396          "subq $128,%%rsp\n\t"                                    \
1397          "movq 32(%%rax), %%rcx\n\t"                              \
1398          "movq 24(%%rax), %%rdx\n\t"                              \
1399          "movq 16(%%rax), %%rsi\n\t"                              \
1400          "movq 8(%%rax), %%rdi\n\t"                               \
1401          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1402          VALGRIND_CALL_NOREDIR_RAX                                \
1403          "addq $128,%%rsp\n\t"                                    \
1404          VALGRIND_CFI_EPILOGUE                                    \
1405          : /*out*/   "=a" (_res)                                  \
1406          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1407          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1408       );                                                          \
1409       lval = (__typeof__(lval)) _res;                             \
1410    } while (0)
1411
1412 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1413    do {                                                           \
1414       volatile OrigFn        _orig = (orig);                      \
1415       volatile unsigned long _argvec[6];                          \
1416       volatile unsigned long _res;                                \
1417       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1418       _argvec[1] = (unsigned long)(arg1);                         \
1419       _argvec[2] = (unsigned long)(arg2);                         \
1420       _argvec[3] = (unsigned long)(arg3);                         \
1421       _argvec[4] = (unsigned long)(arg4);                         \
1422       _argvec[5] = (unsigned long)(arg5);                         \
1423       __asm__ volatile(                                           \
1424          VALGRIND_CFI_PROLOGUE                                    \
1425          "subq $128,%%rsp\n\t"                                    \
1426          "movq 40(%%rax), %%r8\n\t"                               \
1427          "movq 32(%%rax), %%rcx\n\t"                              \
1428          "movq 24(%%rax), %%rdx\n\t"                              \
1429          "movq 16(%%rax), %%rsi\n\t"                              \
1430          "movq 8(%%rax), %%rdi\n\t"                               \
1431          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1432          VALGRIND_CALL_NOREDIR_RAX                                \
1433          "addq $128,%%rsp\n\t"                                    \
1434          VALGRIND_CFI_EPILOGUE                                    \
1435          : /*out*/   "=a" (_res)                                  \
1436          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1437          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1438       );                                                          \
1439       lval = (__typeof__(lval)) _res;                             \
1440    } while (0)
1441
1442 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1443    do {                                                           \
1444       volatile OrigFn        _orig = (orig);                      \
1445       volatile unsigned long _argvec[7];                          \
1446       volatile unsigned long _res;                                \
1447       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1448       _argvec[1] = (unsigned long)(arg1);                         \
1449       _argvec[2] = (unsigned long)(arg2);                         \
1450       _argvec[3] = (unsigned long)(arg3);                         \
1451       _argvec[4] = (unsigned long)(arg4);                         \
1452       _argvec[5] = (unsigned long)(arg5);                         \
1453       _argvec[6] = (unsigned long)(arg6);                         \
1454       __asm__ volatile(                                           \
1455          VALGRIND_CFI_PROLOGUE                                    \
1456          "subq $128,%%rsp\n\t"                                    \
1457          "movq 48(%%rax), %%r9\n\t"                               \
1458          "movq 40(%%rax), %%r8\n\t"                               \
1459          "movq 32(%%rax), %%rcx\n\t"                              \
1460          "movq 24(%%rax), %%rdx\n\t"                              \
1461          "movq 16(%%rax), %%rsi\n\t"                              \
1462          "movq 8(%%rax), %%rdi\n\t"                               \
1463          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1464          VALGRIND_CALL_NOREDIR_RAX                                \
1465          "addq $128,%%rsp\n\t"                                    \
1466          VALGRIND_CFI_EPILOGUE                                    \
1467          : /*out*/   "=a" (_res)                                  \
1468          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1469          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1470       );                                                          \
1471       lval = (__typeof__(lval)) _res;                             \
1472    } while (0)
1473
1474 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1475                                  arg7)                            \
1476    do {                                                           \
1477       volatile OrigFn        _orig = (orig);                      \
1478       volatile unsigned long _argvec[8];                          \
1479       volatile unsigned long _res;                                \
1480       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1481       _argvec[1] = (unsigned long)(arg1);                         \
1482       _argvec[2] = (unsigned long)(arg2);                         \
1483       _argvec[3] = (unsigned long)(arg3);                         \
1484       _argvec[4] = (unsigned long)(arg4);                         \
1485       _argvec[5] = (unsigned long)(arg5);                         \
1486       _argvec[6] = (unsigned long)(arg6);                         \
1487       _argvec[7] = (unsigned long)(arg7);                         \
1488       __asm__ volatile(                                           \
1489          VALGRIND_CFI_PROLOGUE                                    \
1490          "subq $136,%%rsp\n\t"                                    \
1491          "pushq 56(%%rax)\n\t"                                    \
1492          "movq 48(%%rax), %%r9\n\t"                               \
1493          "movq 40(%%rax), %%r8\n\t"                               \
1494          "movq 32(%%rax), %%rcx\n\t"                              \
1495          "movq 24(%%rax), %%rdx\n\t"                              \
1496          "movq 16(%%rax), %%rsi\n\t"                              \
1497          "movq 8(%%rax), %%rdi\n\t"                               \
1498          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1499          VALGRIND_CALL_NOREDIR_RAX                                \
1500          "addq $8, %%rsp\n"                                       \
1501          "addq $136,%%rsp\n\t"                                    \
1502          VALGRIND_CFI_EPILOGUE                                    \
1503          : /*out*/   "=a" (_res)                                  \
1504          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1505          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1506       );                                                          \
1507       lval = (__typeof__(lval)) _res;                             \
1508    } while (0)
1509
1510 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1511                                  arg7,arg8)                       \
1512    do {                                                           \
1513       volatile OrigFn        _orig = (orig);                      \
1514       volatile unsigned long _argvec[9];                          \
1515       volatile unsigned long _res;                                \
1516       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1517       _argvec[1] = (unsigned long)(arg1);                         \
1518       _argvec[2] = (unsigned long)(arg2);                         \
1519       _argvec[3] = (unsigned long)(arg3);                         \
1520       _argvec[4] = (unsigned long)(arg4);                         \
1521       _argvec[5] = (unsigned long)(arg5);                         \
1522       _argvec[6] = (unsigned long)(arg6);                         \
1523       _argvec[7] = (unsigned long)(arg7);                         \
1524       _argvec[8] = (unsigned long)(arg8);                         \
1525       __asm__ volatile(                                           \
1526          VALGRIND_CFI_PROLOGUE                                    \
1527          "subq $128,%%rsp\n\t"                                    \
1528          "pushq 64(%%rax)\n\t"                                    \
1529          "pushq 56(%%rax)\n\t"                                    \
1530          "movq 48(%%rax), %%r9\n\t"                               \
1531          "movq 40(%%rax), %%r8\n\t"                               \
1532          "movq 32(%%rax), %%rcx\n\t"                              \
1533          "movq 24(%%rax), %%rdx\n\t"                              \
1534          "movq 16(%%rax), %%rsi\n\t"                              \
1535          "movq 8(%%rax), %%rdi\n\t"                               \
1536          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1537          VALGRIND_CALL_NOREDIR_RAX                                \
1538          "addq $16, %%rsp\n"                                      \
1539          "addq $128,%%rsp\n\t"                                    \
1540          VALGRIND_CFI_EPILOGUE                                    \
1541          : /*out*/   "=a" (_res)                                  \
1542          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1543          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1544       );                                                          \
1545       lval = (__typeof__(lval)) _res;                             \
1546    } while (0)
1547
1548 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1549                                  arg7,arg8,arg9)                  \
1550    do {                                                           \
1551       volatile OrigFn        _orig = (orig);                      \
1552       volatile unsigned long _argvec[10];                         \
1553       volatile unsigned long _res;                                \
1554       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1555       _argvec[1] = (unsigned long)(arg1);                         \
1556       _argvec[2] = (unsigned long)(arg2);                         \
1557       _argvec[3] = (unsigned long)(arg3);                         \
1558       _argvec[4] = (unsigned long)(arg4);                         \
1559       _argvec[5] = (unsigned long)(arg5);                         \
1560       _argvec[6] = (unsigned long)(arg6);                         \
1561       _argvec[7] = (unsigned long)(arg7);                         \
1562       _argvec[8] = (unsigned long)(arg8);                         \
1563       _argvec[9] = (unsigned long)(arg9);                         \
1564       __asm__ volatile(                                           \
1565          VALGRIND_CFI_PROLOGUE                                    \
1566          "subq $136,%%rsp\n\t"                                    \
1567          "pushq 72(%%rax)\n\t"                                    \
1568          "pushq 64(%%rax)\n\t"                                    \
1569          "pushq 56(%%rax)\n\t"                                    \
1570          "movq 48(%%rax), %%r9\n\t"                               \
1571          "movq 40(%%rax), %%r8\n\t"                               \
1572          "movq 32(%%rax), %%rcx\n\t"                              \
1573          "movq 24(%%rax), %%rdx\n\t"                              \
1574          "movq 16(%%rax), %%rsi\n\t"                              \
1575          "movq 8(%%rax), %%rdi\n\t"                               \
1576          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1577          VALGRIND_CALL_NOREDIR_RAX                                \
1578          "addq $24, %%rsp\n"                                      \
1579          "addq $136,%%rsp\n\t"                                    \
1580          VALGRIND_CFI_EPILOGUE                                    \
1581          : /*out*/   "=a" (_res)                                  \
1582          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1583          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1584       );                                                          \
1585       lval = (__typeof__(lval)) _res;                             \
1586    } while (0)
1587
1588 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1589                                   arg7,arg8,arg9,arg10)           \
1590    do {                                                           \
1591       volatile OrigFn        _orig = (orig);                      \
1592       volatile unsigned long _argvec[11];                         \
1593       volatile unsigned long _res;                                \
1594       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1595       _argvec[1] = (unsigned long)(arg1);                         \
1596       _argvec[2] = (unsigned long)(arg2);                         \
1597       _argvec[3] = (unsigned long)(arg3);                         \
1598       _argvec[4] = (unsigned long)(arg4);                         \
1599       _argvec[5] = (unsigned long)(arg5);                         \
1600       _argvec[6] = (unsigned long)(arg6);                         \
1601       _argvec[7] = (unsigned long)(arg7);                         \
1602       _argvec[8] = (unsigned long)(arg8);                         \
1603       _argvec[9] = (unsigned long)(arg9);                         \
1604       _argvec[10] = (unsigned long)(arg10);                       \
1605       __asm__ volatile(                                           \
1606          VALGRIND_CFI_PROLOGUE                                    \
1607          "subq $128,%%rsp\n\t"                                    \
1608          "pushq 80(%%rax)\n\t"                                    \
1609          "pushq 72(%%rax)\n\t"                                    \
1610          "pushq 64(%%rax)\n\t"                                    \
1611          "pushq 56(%%rax)\n\t"                                    \
1612          "movq 48(%%rax), %%r9\n\t"                               \
1613          "movq 40(%%rax), %%r8\n\t"                               \
1614          "movq 32(%%rax), %%rcx\n\t"                              \
1615          "movq 24(%%rax), %%rdx\n\t"                              \
1616          "movq 16(%%rax), %%rsi\n\t"                              \
1617          "movq 8(%%rax), %%rdi\n\t"                               \
1618          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1619          VALGRIND_CALL_NOREDIR_RAX                                \
1620          "addq $32, %%rsp\n"                                      \
1621          "addq $128,%%rsp\n\t"                                    \
1622          VALGRIND_CFI_EPILOGUE                                    \
1623          : /*out*/   "=a" (_res)                                  \
1624          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1625          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1626       );                                                          \
1627       lval = (__typeof__(lval)) _res;                             \
1628    } while (0)
1629
1630 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1631                                   arg7,arg8,arg9,arg10,arg11)     \
1632    do {                                                           \
1633       volatile OrigFn        _orig = (orig);                      \
1634       volatile unsigned long _argvec[12];                         \
1635       volatile unsigned long _res;                                \
1636       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1637       _argvec[1] = (unsigned long)(arg1);                         \
1638       _argvec[2] = (unsigned long)(arg2);                         \
1639       _argvec[3] = (unsigned long)(arg3);                         \
1640       _argvec[4] = (unsigned long)(arg4);                         \
1641       _argvec[5] = (unsigned long)(arg5);                         \
1642       _argvec[6] = (unsigned long)(arg6);                         \
1643       _argvec[7] = (unsigned long)(arg7);                         \
1644       _argvec[8] = (unsigned long)(arg8);                         \
1645       _argvec[9] = (unsigned long)(arg9);                         \
1646       _argvec[10] = (unsigned long)(arg10);                       \
1647       _argvec[11] = (unsigned long)(arg11);                       \
1648       __asm__ volatile(                                           \
1649          VALGRIND_CFI_PROLOGUE                                    \
1650          "subq $136,%%rsp\n\t"                                    \
1651          "pushq 88(%%rax)\n\t"                                    \
1652          "pushq 80(%%rax)\n\t"                                    \
1653          "pushq 72(%%rax)\n\t"                                    \
1654          "pushq 64(%%rax)\n\t"                                    \
1655          "pushq 56(%%rax)\n\t"                                    \
1656          "movq 48(%%rax), %%r9\n\t"                               \
1657          "movq 40(%%rax), %%r8\n\t"                               \
1658          "movq 32(%%rax), %%rcx\n\t"                              \
1659          "movq 24(%%rax), %%rdx\n\t"                              \
1660          "movq 16(%%rax), %%rsi\n\t"                              \
1661          "movq 8(%%rax), %%rdi\n\t"                               \
1662          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1663          VALGRIND_CALL_NOREDIR_RAX                                \
1664          "addq $40, %%rsp\n"                                      \
1665          "addq $136,%%rsp\n\t"                                    \
1666          VALGRIND_CFI_EPILOGUE                                    \
1667          : /*out*/   "=a" (_res)                                  \
1668          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1669          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1670       );                                                          \
1671       lval = (__typeof__(lval)) _res;                             \
1672    } while (0)
1673
1674 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1675                                 arg7,arg8,arg9,arg10,arg11,arg12) \
1676    do {                                                           \
1677       volatile OrigFn        _orig = (orig);                      \
1678       volatile unsigned long _argvec[13];                         \
1679       volatile unsigned long _res;                                \
1680       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1681       _argvec[1] = (unsigned long)(arg1);                         \
1682       _argvec[2] = (unsigned long)(arg2);                         \
1683       _argvec[3] = (unsigned long)(arg3);                         \
1684       _argvec[4] = (unsigned long)(arg4);                         \
1685       _argvec[5] = (unsigned long)(arg5);                         \
1686       _argvec[6] = (unsigned long)(arg6);                         \
1687       _argvec[7] = (unsigned long)(arg7);                         \
1688       _argvec[8] = (unsigned long)(arg8);                         \
1689       _argvec[9] = (unsigned long)(arg9);                         \
1690       _argvec[10] = (unsigned long)(arg10);                       \
1691       _argvec[11] = (unsigned long)(arg11);                       \
1692       _argvec[12] = (unsigned long)(arg12);                       \
1693       __asm__ volatile(                                           \
1694          VALGRIND_CFI_PROLOGUE                                    \
1695          "subq $128,%%rsp\n\t"                                    \
1696          "pushq 96(%%rax)\n\t"                                    \
1697          "pushq 88(%%rax)\n\t"                                    \
1698          "pushq 80(%%rax)\n\t"                                    \
1699          "pushq 72(%%rax)\n\t"                                    \
1700          "pushq 64(%%rax)\n\t"                                    \
1701          "pushq 56(%%rax)\n\t"                                    \
1702          "movq 48(%%rax), %%r9\n\t"                               \
1703          "movq 40(%%rax), %%r8\n\t"                               \
1704          "movq 32(%%rax), %%rcx\n\t"                              \
1705          "movq 24(%%rax), %%rdx\n\t"                              \
1706          "movq 16(%%rax), %%rsi\n\t"                              \
1707          "movq 8(%%rax), %%rdi\n\t"                               \
1708          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1709          VALGRIND_CALL_NOREDIR_RAX                                \
1710          "addq $48, %%rsp\n"                                      \
1711          "addq $128,%%rsp\n\t"                                    \
1712          VALGRIND_CFI_EPILOGUE                                    \
1713          : /*out*/   "=a" (_res)                                  \
1714          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1715          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1716       );                                                          \
1717       lval = (__typeof__(lval)) _res;                             \
1718    } while (0)
1719
1720 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
1721
1722 /* ------------------------ ppc32-linux ------------------------ */
1723
1724 #if defined(PLAT_ppc32_linux)
1725
1726 /* This is useful for finding out about the on-stack stuff:
1727
1728    extern int f9  ( int,int,int,int,int,int,int,int,int );
1729    extern int f10 ( int,int,int,int,int,int,int,int,int,int );
1730    extern int f11 ( int,int,int,int,int,int,int,int,int,int,int );
1731    extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int );
1732
1733    int g9 ( void ) {
1734       return f9(11,22,33,44,55,66,77,88,99);
1735    }
1736    int g10 ( void ) {
1737       return f10(11,22,33,44,55,66,77,88,99,110);
1738    }
1739    int g11 ( void ) {
1740       return f11(11,22,33,44,55,66,77,88,99,110,121);
1741    }
1742    int g12 ( void ) {
1743       return f12(11,22,33,44,55,66,77,88,99,110,121,132);
1744    }
1745 */
1746
1747 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
1748
1749 /* These regs are trashed by the hidden call. */
1750 #define __CALLER_SAVED_REGS                                       \
1751    "lr", "ctr", "xer",                                            \
1752    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
1753    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
1754    "r11", "r12", "r13"
1755
1756 /* These CALL_FN_ macros assume that on ppc32-linux,
1757    sizeof(unsigned long) == 4. */
1758
1759 #define CALL_FN_W_v(lval, orig)                                   \
1760    do {                                                           \
1761       volatile OrigFn        _orig = (orig);                      \
1762       volatile unsigned long _argvec[1];                          \
1763       volatile unsigned long _res;                                \
1764       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1765       __asm__ volatile(                                           \
1766          "mr 11,%1\n\t"                                           \
1767          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1768          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1769          "mr %0,3"                                                \
1770          : /*out*/   "=r" (_res)                                  \
1771          : /*in*/    "r" (&_argvec[0])                            \
1772          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1773       );                                                          \
1774       lval = (__typeof__(lval)) _res;                             \
1775    } while (0)
1776
1777 #define CALL_FN_W_W(lval, orig, arg1)                             \
1778    do {                                                           \
1779       volatile OrigFn        _orig = (orig);                      \
1780       volatile unsigned long _argvec[2];                          \
1781       volatile unsigned long _res;                                \
1782       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1783       _argvec[1] = (unsigned long)arg1;                           \
1784       __asm__ volatile(                                           \
1785          "mr 11,%1\n\t"                                           \
1786          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1787          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1788          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1789          "mr %0,3"                                                \
1790          : /*out*/   "=r" (_res)                                  \
1791          : /*in*/    "r" (&_argvec[0])                            \
1792          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1793       );                                                          \
1794       lval = (__typeof__(lval)) _res;                             \
1795    } while (0)
1796
1797 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1798    do {                                                           \
1799       volatile OrigFn        _orig = (orig);                      \
1800       volatile unsigned long _argvec[3];                          \
1801       volatile unsigned long _res;                                \
1802       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1803       _argvec[1] = (unsigned long)arg1;                           \
1804       _argvec[2] = (unsigned long)arg2;                           \
1805       __asm__ volatile(                                           \
1806          "mr 11,%1\n\t"                                           \
1807          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1808          "lwz 4,8(11)\n\t"                                        \
1809          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1810          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1811          "mr %0,3"                                                \
1812          : /*out*/   "=r" (_res)                                  \
1813          : /*in*/    "r" (&_argvec[0])                            \
1814          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1815       );                                                          \
1816       lval = (__typeof__(lval)) _res;                             \
1817    } while (0)
1818
1819 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1820    do {                                                           \
1821       volatile OrigFn        _orig = (orig);                      \
1822       volatile unsigned long _argvec[4];                          \
1823       volatile unsigned long _res;                                \
1824       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1825       _argvec[1] = (unsigned long)arg1;                           \
1826       _argvec[2] = (unsigned long)arg2;                           \
1827       _argvec[3] = (unsigned long)arg3;                           \
1828       __asm__ volatile(                                           \
1829          "mr 11,%1\n\t"                                           \
1830          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1831          "lwz 4,8(11)\n\t"                                        \
1832          "lwz 5,12(11)\n\t"                                       \
1833          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1834          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1835          "mr %0,3"                                                \
1836          : /*out*/   "=r" (_res)                                  \
1837          : /*in*/    "r" (&_argvec[0])                            \
1838          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1839       );                                                          \
1840       lval = (__typeof__(lval)) _res;                             \
1841    } while (0)
1842
1843 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1844    do {                                                           \
1845       volatile OrigFn        _orig = (orig);                      \
1846       volatile unsigned long _argvec[5];                          \
1847       volatile unsigned long _res;                                \
1848       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1849       _argvec[1] = (unsigned long)arg1;                           \
1850       _argvec[2] = (unsigned long)arg2;                           \
1851       _argvec[3] = (unsigned long)arg3;                           \
1852       _argvec[4] = (unsigned long)arg4;                           \
1853       __asm__ volatile(                                           \
1854          "mr 11,%1\n\t"                                           \
1855          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1856          "lwz 4,8(11)\n\t"                                        \
1857          "lwz 5,12(11)\n\t"                                       \
1858          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1859          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1860          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1861          "mr %0,3"                                                \
1862          : /*out*/   "=r" (_res)                                  \
1863          : /*in*/    "r" (&_argvec[0])                            \
1864          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1865       );                                                          \
1866       lval = (__typeof__(lval)) _res;                             \
1867    } while (0)
1868
1869 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1870    do {                                                           \
1871       volatile OrigFn        _orig = (orig);                      \
1872       volatile unsigned long _argvec[6];                          \
1873       volatile unsigned long _res;                                \
1874       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1875       _argvec[1] = (unsigned long)arg1;                           \
1876       _argvec[2] = (unsigned long)arg2;                           \
1877       _argvec[3] = (unsigned long)arg3;                           \
1878       _argvec[4] = (unsigned long)arg4;                           \
1879       _argvec[5] = (unsigned long)arg5;                           \
1880       __asm__ volatile(                                           \
1881          "mr 11,%1\n\t"                                           \
1882          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1883          "lwz 4,8(11)\n\t"                                        \
1884          "lwz 5,12(11)\n\t"                                       \
1885          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1886          "lwz 7,20(11)\n\t"                                       \
1887          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1888          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1889          "mr %0,3"                                                \
1890          : /*out*/   "=r" (_res)                                  \
1891          : /*in*/    "r" (&_argvec[0])                            \
1892          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1893       );                                                          \
1894       lval = (__typeof__(lval)) _res;                             \
1895    } while (0)
1896
1897 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1898    do {                                                           \
1899       volatile OrigFn        _orig = (orig);                      \
1900       volatile unsigned long _argvec[7];                          \
1901       volatile unsigned long _res;                                \
1902       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1903       _argvec[1] = (unsigned long)arg1;                           \
1904       _argvec[2] = (unsigned long)arg2;                           \
1905       _argvec[3] = (unsigned long)arg3;                           \
1906       _argvec[4] = (unsigned long)arg4;                           \
1907       _argvec[5] = (unsigned long)arg5;                           \
1908       _argvec[6] = (unsigned long)arg6;                           \
1909       __asm__ volatile(                                           \
1910          "mr 11,%1\n\t"                                           \
1911          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1912          "lwz 4,8(11)\n\t"                                        \
1913          "lwz 5,12(11)\n\t"                                       \
1914          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1915          "lwz 7,20(11)\n\t"                                       \
1916          "lwz 8,24(11)\n\t"                                       \
1917          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1918          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1919          "mr %0,3"                                                \
1920          : /*out*/   "=r" (_res)                                  \
1921          : /*in*/    "r" (&_argvec[0])                            \
1922          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1923       );                                                          \
1924       lval = (__typeof__(lval)) _res;                             \
1925    } while (0)
1926
1927 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1928                                  arg7)                            \
1929    do {                                                           \
1930       volatile OrigFn        _orig = (orig);                      \
1931       volatile unsigned long _argvec[8];                          \
1932       volatile unsigned long _res;                                \
1933       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1934       _argvec[1] = (unsigned long)arg1;                           \
1935       _argvec[2] = (unsigned long)arg2;                           \
1936       _argvec[3] = (unsigned long)arg3;                           \
1937       _argvec[4] = (unsigned long)arg4;                           \
1938       _argvec[5] = (unsigned long)arg5;                           \
1939       _argvec[6] = (unsigned long)arg6;                           \
1940       _argvec[7] = (unsigned long)arg7;                           \
1941       __asm__ volatile(                                           \
1942          "mr 11,%1\n\t"                                           \
1943          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1944          "lwz 4,8(11)\n\t"                                        \
1945          "lwz 5,12(11)\n\t"                                       \
1946          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1947          "lwz 7,20(11)\n\t"                                       \
1948          "lwz 8,24(11)\n\t"                                       \
1949          "lwz 9,28(11)\n\t"                                       \
1950          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1951          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1952          "mr %0,3"                                                \
1953          : /*out*/   "=r" (_res)                                  \
1954          : /*in*/    "r" (&_argvec[0])                            \
1955          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1956       );                                                          \
1957       lval = (__typeof__(lval)) _res;                             \
1958    } while (0)
1959
1960 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1961                                  arg7,arg8)                       \
1962    do {                                                           \
1963       volatile OrigFn        _orig = (orig);                      \
1964       volatile unsigned long _argvec[9];                          \
1965       volatile unsigned long _res;                                \
1966       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1967       _argvec[1] = (unsigned long)arg1;                           \
1968       _argvec[2] = (unsigned long)arg2;                           \
1969       _argvec[3] = (unsigned long)arg3;                           \
1970       _argvec[4] = (unsigned long)arg4;                           \
1971       _argvec[5] = (unsigned long)arg5;                           \
1972       _argvec[6] = (unsigned long)arg6;                           \
1973       _argvec[7] = (unsigned long)arg7;                           \
1974       _argvec[8] = (unsigned long)arg8;                           \
1975       __asm__ volatile(                                           \
1976          "mr 11,%1\n\t"                                           \
1977          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1978          "lwz 4,8(11)\n\t"                                        \
1979          "lwz 5,12(11)\n\t"                                       \
1980          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1981          "lwz 7,20(11)\n\t"                                       \
1982          "lwz 8,24(11)\n\t"                                       \
1983          "lwz 9,28(11)\n\t"                                       \
1984          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1985          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1986          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1987          "mr %0,3"                                                \
1988          : /*out*/   "=r" (_res)                                  \
1989          : /*in*/    "r" (&_argvec[0])                            \
1990          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1991       );                                                          \
1992       lval = (__typeof__(lval)) _res;                             \
1993    } while (0)
1994
1995 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1996                                  arg7,arg8,arg9)                  \
1997    do {                                                           \
1998       volatile OrigFn        _orig = (orig);                      \
1999       volatile unsigned long _argvec[10];                         \
2000       volatile unsigned long _res;                                \
2001       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2002       _argvec[1] = (unsigned long)arg1;                           \
2003       _argvec[2] = (unsigned long)arg2;                           \
2004       _argvec[3] = (unsigned long)arg3;                           \
2005       _argvec[4] = (unsigned long)arg4;                           \
2006       _argvec[5] = (unsigned long)arg5;                           \
2007       _argvec[6] = (unsigned long)arg6;                           \
2008       _argvec[7] = (unsigned long)arg7;                           \
2009       _argvec[8] = (unsigned long)arg8;                           \
2010       _argvec[9] = (unsigned long)arg9;                           \
2011       __asm__ volatile(                                           \
2012          "mr 11,%1\n\t"                                           \
2013          "addi 1,1,-16\n\t"                                       \
2014          /* arg9 */                                               \
2015          "lwz 3,36(11)\n\t"                                       \
2016          "stw 3,8(1)\n\t"                                         \
2017          /* args1-8 */                                            \
2018          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2019          "lwz 4,8(11)\n\t"                                        \
2020          "lwz 5,12(11)\n\t"                                       \
2021          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2022          "lwz 7,20(11)\n\t"                                       \
2023          "lwz 8,24(11)\n\t"                                       \
2024          "lwz 9,28(11)\n\t"                                       \
2025          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2026          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2027          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2028          "addi 1,1,16\n\t"                                        \
2029          "mr %0,3"                                                \
2030          : /*out*/   "=r" (_res)                                  \
2031          : /*in*/    "r" (&_argvec[0])                            \
2032          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2033       );                                                          \
2034       lval = (__typeof__(lval)) _res;                             \
2035    } while (0)
2036
2037 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2038                                   arg7,arg8,arg9,arg10)           \
2039    do {                                                           \
2040       volatile OrigFn        _orig = (orig);                      \
2041       volatile unsigned long _argvec[11];                         \
2042       volatile unsigned long _res;                                \
2043       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2044       _argvec[1] = (unsigned long)arg1;                           \
2045       _argvec[2] = (unsigned long)arg2;                           \
2046       _argvec[3] = (unsigned long)arg3;                           \
2047       _argvec[4] = (unsigned long)arg4;                           \
2048       _argvec[5] = (unsigned long)arg5;                           \
2049       _argvec[6] = (unsigned long)arg6;                           \
2050       _argvec[7] = (unsigned long)arg7;                           \
2051       _argvec[8] = (unsigned long)arg8;                           \
2052       _argvec[9] = (unsigned long)arg9;                           \
2053       _argvec[10] = (unsigned long)arg10;                         \
2054       __asm__ volatile(                                           \
2055          "mr 11,%1\n\t"                                           \
2056          "addi 1,1,-16\n\t"                                       \
2057          /* arg10 */                                              \
2058          "lwz 3,40(11)\n\t"                                       \
2059          "stw 3,12(1)\n\t"                                        \
2060          /* arg9 */                                               \
2061          "lwz 3,36(11)\n\t"                                       \
2062          "stw 3,8(1)\n\t"                                         \
2063          /* args1-8 */                                            \
2064          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2065          "lwz 4,8(11)\n\t"                                        \
2066          "lwz 5,12(11)\n\t"                                       \
2067          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2068          "lwz 7,20(11)\n\t"                                       \
2069          "lwz 8,24(11)\n\t"                                       \
2070          "lwz 9,28(11)\n\t"                                       \
2071          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2072          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2073          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2074          "addi 1,1,16\n\t"                                        \
2075          "mr %0,3"                                                \
2076          : /*out*/   "=r" (_res)                                  \
2077          : /*in*/    "r" (&_argvec[0])                            \
2078          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2079       );                                                          \
2080       lval = (__typeof__(lval)) _res;                             \
2081    } while (0)
2082
2083 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2084                                   arg7,arg8,arg9,arg10,arg11)     \
2085    do {                                                           \
2086       volatile OrigFn        _orig = (orig);                      \
2087       volatile unsigned long _argvec[12];                         \
2088       volatile unsigned long _res;                                \
2089       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2090       _argvec[1] = (unsigned long)arg1;                           \
2091       _argvec[2] = (unsigned long)arg2;                           \
2092       _argvec[3] = (unsigned long)arg3;                           \
2093       _argvec[4] = (unsigned long)arg4;                           \
2094       _argvec[5] = (unsigned long)arg5;                           \
2095       _argvec[6] = (unsigned long)arg6;                           \
2096       _argvec[7] = (unsigned long)arg7;                           \
2097       _argvec[8] = (unsigned long)arg8;                           \
2098       _argvec[9] = (unsigned long)arg9;                           \
2099       _argvec[10] = (unsigned long)arg10;                         \
2100       _argvec[11] = (unsigned long)arg11;                         \
2101       __asm__ volatile(                                           \
2102          "mr 11,%1\n\t"                                           \
2103          "addi 1,1,-32\n\t"                                       \
2104          /* arg11 */                                              \
2105          "lwz 3,44(11)\n\t"                                       \
2106          "stw 3,16(1)\n\t"                                        \
2107          /* arg10 */                                              \
2108          "lwz 3,40(11)\n\t"                                       \
2109          "stw 3,12(1)\n\t"                                        \
2110          /* arg9 */                                               \
2111          "lwz 3,36(11)\n\t"                                       \
2112          "stw 3,8(1)\n\t"                                         \
2113          /* args1-8 */                                            \
2114          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2115          "lwz 4,8(11)\n\t"                                        \
2116          "lwz 5,12(11)\n\t"                                       \
2117          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2118          "lwz 7,20(11)\n\t"                                       \
2119          "lwz 8,24(11)\n\t"                                       \
2120          "lwz 9,28(11)\n\t"                                       \
2121          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2122          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2123          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2124          "addi 1,1,32\n\t"                                        \
2125          "mr %0,3"                                                \
2126          : /*out*/   "=r" (_res)                                  \
2127          : /*in*/    "r" (&_argvec[0])                            \
2128          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2129       );                                                          \
2130       lval = (__typeof__(lval)) _res;                             \
2131    } while (0)
2132
2133 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2134                                 arg7,arg8,arg9,arg10,arg11,arg12) \
2135    do {                                                           \
2136       volatile OrigFn        _orig = (orig);                      \
2137       volatile unsigned long _argvec[13];                         \
2138       volatile unsigned long _res;                                \
2139       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2140       _argvec[1] = (unsigned long)arg1;                           \
2141       _argvec[2] = (unsigned long)arg2;                           \
2142       _argvec[3] = (unsigned long)arg3;                           \
2143       _argvec[4] = (unsigned long)arg4;                           \
2144       _argvec[5] = (unsigned long)arg5;                           \
2145       _argvec[6] = (unsigned long)arg6;                           \
2146       _argvec[7] = (unsigned long)arg7;                           \
2147       _argvec[8] = (unsigned long)arg8;                           \
2148       _argvec[9] = (unsigned long)arg9;                           \
2149       _argvec[10] = (unsigned long)arg10;                         \
2150       _argvec[11] = (unsigned long)arg11;                         \
2151       _argvec[12] = (unsigned long)arg12;                         \
2152       __asm__ volatile(                                           \
2153          "mr 11,%1\n\t"                                           \
2154          "addi 1,1,-32\n\t"                                       \
2155          /* arg12 */                                              \
2156          "lwz 3,48(11)\n\t"                                       \
2157          "stw 3,20(1)\n\t"                                        \
2158          /* arg11 */                                              \
2159          "lwz 3,44(11)\n\t"                                       \
2160          "stw 3,16(1)\n\t"                                        \
2161          /* arg10 */                                              \
2162          "lwz 3,40(11)\n\t"                                       \
2163          "stw 3,12(1)\n\t"                                        \
2164          /* arg9 */                                               \
2165          "lwz 3,36(11)\n\t"                                       \
2166          "stw 3,8(1)\n\t"                                         \
2167          /* args1-8 */                                            \
2168          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2169          "lwz 4,8(11)\n\t"                                        \
2170          "lwz 5,12(11)\n\t"                                       \
2171          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2172          "lwz 7,20(11)\n\t"                                       \
2173          "lwz 8,24(11)\n\t"                                       \
2174          "lwz 9,28(11)\n\t"                                       \
2175          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2176          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2177          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2178          "addi 1,1,32\n\t"                                        \
2179          "mr %0,3"                                                \
2180          : /*out*/   "=r" (_res)                                  \
2181          : /*in*/    "r" (&_argvec[0])                            \
2182          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2183       );                                                          \
2184       lval = (__typeof__(lval)) _res;                             \
2185    } while (0)
2186
2187 #endif /* PLAT_ppc32_linux */
2188
2189 /* ------------------------ ppc64-linux ------------------------ */
2190
2191 #if defined(PLAT_ppc64_linux)
2192
2193 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2194
2195 /* These regs are trashed by the hidden call. */
2196 #define __CALLER_SAVED_REGS                                       \
2197    "lr", "ctr", "xer",                                            \
2198    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
2199    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
2200    "r11", "r12", "r13"
2201
2202 /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
2203    long) == 8. */
2204
2205 #define CALL_FN_W_v(lval, orig)                                   \
2206    do {                                                           \
2207       volatile OrigFn        _orig = (orig);                      \
2208       volatile unsigned long _argvec[3+0];                        \
2209       volatile unsigned long _res;                                \
2210       /* _argvec[0] holds current r2 across the call */           \
2211       _argvec[1] = (unsigned long)_orig.r2;                       \
2212       _argvec[2] = (unsigned long)_orig.nraddr;                   \
2213       __asm__ volatile(                                           \
2214          "mr 11,%1\n\t"                                           \
2215          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2216          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2217          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2218          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2219          "mr 11,%1\n\t"                                           \
2220          "mr %0,3\n\t"                                            \
2221          "ld 2,-16(11)" /* restore tocptr */                      \
2222          : /*out*/   "=r" (_res)                                  \
2223          : /*in*/    "r" (&_argvec[2])                            \
2224          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2225       );                                                          \
2226       lval = (__typeof__(lval)) _res;                             \
2227    } while (0)
2228
2229 #define CALL_FN_W_W(lval, orig, arg1)                             \
2230    do {                                                           \
2231       volatile OrigFn        _orig = (orig);                      \
2232       volatile unsigned long _argvec[3+1];                        \
2233       volatile unsigned long _res;                                \
2234       /* _argvec[0] holds current r2 across the call */           \
2235       _argvec[1]   = (unsigned long)_orig.r2;                     \
2236       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2237       _argvec[2+1] = (unsigned long)arg1;                         \
2238       __asm__ volatile(                                           \
2239          "mr 11,%1\n\t"                                           \
2240          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2241          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2242          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2243          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2244          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2245          "mr 11,%1\n\t"                                           \
2246          "mr %0,3\n\t"                                            \
2247          "ld 2,-16(11)" /* restore tocptr */                      \
2248          : /*out*/   "=r" (_res)                                  \
2249          : /*in*/    "r" (&_argvec[2])                            \
2250          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2251       );                                                          \
2252       lval = (__typeof__(lval)) _res;                             \
2253    } while (0)
2254
2255 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2256    do {                                                           \
2257       volatile OrigFn        _orig = (orig);                      \
2258       volatile unsigned long _argvec[3+2];                        \
2259       volatile unsigned long _res;                                \
2260       /* _argvec[0] holds current r2 across the call */           \
2261       _argvec[1]   = (unsigned long)_orig.r2;                     \
2262       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2263       _argvec[2+1] = (unsigned long)arg1;                         \
2264       _argvec[2+2] = (unsigned long)arg2;                         \
2265       __asm__ volatile(                                           \
2266          "mr 11,%1\n\t"                                           \
2267          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2268          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2269          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2270          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2271          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2272          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2273          "mr 11,%1\n\t"                                           \
2274          "mr %0,3\n\t"                                            \
2275          "ld 2,-16(11)" /* restore tocptr */                      \
2276          : /*out*/   "=r" (_res)                                  \
2277          : /*in*/    "r" (&_argvec[2])                            \
2278          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2279       );                                                          \
2280       lval = (__typeof__(lval)) _res;                             \
2281    } while (0)
2282
2283 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2284    do {                                                           \
2285       volatile OrigFn        _orig = (orig);                      \
2286       volatile unsigned long _argvec[3+3];                        \
2287       volatile unsigned long _res;                                \
2288       /* _argvec[0] holds current r2 across the call */           \
2289       _argvec[1]   = (unsigned long)_orig.r2;                     \
2290       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2291       _argvec[2+1] = (unsigned long)arg1;                         \
2292       _argvec[2+2] = (unsigned long)arg2;                         \
2293       _argvec[2+3] = (unsigned long)arg3;                         \
2294       __asm__ volatile(                                           \
2295          "mr 11,%1\n\t"                                           \
2296          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2297          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2298          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2299          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2300          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2301          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2302          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2303          "mr 11,%1\n\t"                                           \
2304          "mr %0,3\n\t"                                            \
2305          "ld 2,-16(11)" /* restore tocptr */                      \
2306          : /*out*/   "=r" (_res)                                  \
2307          : /*in*/    "r" (&_argvec[2])                            \
2308          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2309       );                                                          \
2310       lval = (__typeof__(lval)) _res;                             \
2311    } while (0)
2312
2313 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2314    do {                                                           \
2315       volatile OrigFn        _orig = (orig);                      \
2316       volatile unsigned long _argvec[3+4];                        \
2317       volatile unsigned long _res;                                \
2318       /* _argvec[0] holds current r2 across the call */           \
2319       _argvec[1]   = (unsigned long)_orig.r2;                     \
2320       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2321       _argvec[2+1] = (unsigned long)arg1;                         \
2322       _argvec[2+2] = (unsigned long)arg2;                         \
2323       _argvec[2+3] = (unsigned long)arg3;                         \
2324       _argvec[2+4] = (unsigned long)arg4;                         \
2325       __asm__ volatile(                                           \
2326          "mr 11,%1\n\t"                                           \
2327          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2328          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2329          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2330          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2331          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2332          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2333          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2334          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2335          "mr 11,%1\n\t"                                           \
2336          "mr %0,3\n\t"                                            \
2337          "ld 2,-16(11)" /* restore tocptr */                      \
2338          : /*out*/   "=r" (_res)                                  \
2339          : /*in*/    "r" (&_argvec[2])                            \
2340          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2341       );                                                          \
2342       lval = (__typeof__(lval)) _res;                             \
2343    } while (0)
2344
2345 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2346    do {                                                           \
2347       volatile OrigFn        _orig = (orig);                      \
2348       volatile unsigned long _argvec[3+5];                        \
2349       volatile unsigned long _res;                                \
2350       /* _argvec[0] holds current r2 across the call */           \
2351       _argvec[1]   = (unsigned long)_orig.r2;                     \
2352       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2353       _argvec[2+1] = (unsigned long)arg1;                         \
2354       _argvec[2+2] = (unsigned long)arg2;                         \
2355       _argvec[2+3] = (unsigned long)arg3;                         \
2356       _argvec[2+4] = (unsigned long)arg4;                         \
2357       _argvec[2+5] = (unsigned long)arg5;                         \
2358       __asm__ volatile(                                           \
2359          "mr 11,%1\n\t"                                           \
2360          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2361          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2362          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2363          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2364          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2365          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2366          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2367          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2368          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2369          "mr 11,%1\n\t"                                           \
2370          "mr %0,3\n\t"                                            \
2371          "ld 2,-16(11)" /* restore tocptr */                      \
2372          : /*out*/   "=r" (_res)                                  \
2373          : /*in*/    "r" (&_argvec[2])                            \
2374          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2375       );                                                          \
2376       lval = (__typeof__(lval)) _res;                             \
2377    } while (0)
2378
2379 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2380    do {                                                           \
2381       volatile OrigFn        _orig = (orig);                      \
2382       volatile unsigned long _argvec[3+6];                        \
2383       volatile unsigned long _res;                                \
2384       /* _argvec[0] holds current r2 across the call */           \
2385       _argvec[1]   = (unsigned long)_orig.r2;                     \
2386       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2387       _argvec[2+1] = (unsigned long)arg1;                         \
2388       _argvec[2+2] = (unsigned long)arg2;                         \
2389       _argvec[2+3] = (unsigned long)arg3;                         \
2390       _argvec[2+4] = (unsigned long)arg4;                         \
2391       _argvec[2+5] = (unsigned long)arg5;                         \
2392       _argvec[2+6] = (unsigned long)arg6;                         \
2393       __asm__ volatile(                                           \
2394          "mr 11,%1\n\t"                                           \
2395          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2396          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2397          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2398          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2399          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2400          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2401          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2402          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2403          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2404          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2405          "mr 11,%1\n\t"                                           \
2406          "mr %0,3\n\t"                                            \
2407          "ld 2,-16(11)" /* restore tocptr */                      \
2408          : /*out*/   "=r" (_res)                                  \
2409          : /*in*/    "r" (&_argvec[2])                            \
2410          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2411       );                                                          \
2412       lval = (__typeof__(lval)) _res;                             \
2413    } while (0)
2414
2415 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2416                                  arg7)                            \
2417    do {                                                           \
2418       volatile OrigFn        _orig = (orig);                      \
2419       volatile unsigned long _argvec[3+7];                        \
2420       volatile unsigned long _res;                                \
2421       /* _argvec[0] holds current r2 across the call */           \
2422       _argvec[1]   = (unsigned long)_orig.r2;                     \
2423       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2424       _argvec[2+1] = (unsigned long)arg1;                         \
2425       _argvec[2+2] = (unsigned long)arg2;                         \
2426       _argvec[2+3] = (unsigned long)arg3;                         \
2427       _argvec[2+4] = (unsigned long)arg4;                         \
2428       _argvec[2+5] = (unsigned long)arg5;                         \
2429       _argvec[2+6] = (unsigned long)arg6;                         \
2430       _argvec[2+7] = (unsigned long)arg7;                         \
2431       __asm__ volatile(                                           \
2432          "mr 11,%1\n\t"                                           \
2433          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2434          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2435          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2436          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2437          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2438          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2439          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2440          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2441          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2442          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2443          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2444          "mr 11,%1\n\t"                                           \
2445          "mr %0,3\n\t"                                            \
2446          "ld 2,-16(11)" /* restore tocptr */                      \
2447          : /*out*/   "=r" (_res)                                  \
2448          : /*in*/    "r" (&_argvec[2])                            \
2449          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2450       );                                                          \
2451       lval = (__typeof__(lval)) _res;                             \
2452    } while (0)
2453
2454 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2455                                  arg7,arg8)                       \
2456    do {                                                           \
2457       volatile OrigFn        _orig = (orig);                      \
2458       volatile unsigned long _argvec[3+8];                        \
2459       volatile unsigned long _res;                                \
2460       /* _argvec[0] holds current r2 across the call */           \
2461       _argvec[1]   = (unsigned long)_orig.r2;                     \
2462       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2463       _argvec[2+1] = (unsigned long)arg1;                         \
2464       _argvec[2+2] = (unsigned long)arg2;                         \
2465       _argvec[2+3] = (unsigned long)arg3;                         \
2466       _argvec[2+4] = (unsigned long)arg4;                         \
2467       _argvec[2+5] = (unsigned long)arg5;                         \
2468       _argvec[2+6] = (unsigned long)arg6;                         \
2469       _argvec[2+7] = (unsigned long)arg7;                         \
2470       _argvec[2+8] = (unsigned long)arg8;                         \
2471       __asm__ volatile(                                           \
2472          "mr 11,%1\n\t"                                           \
2473          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2474          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2475          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2476          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2477          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2478          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2479          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2480          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2481          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2482          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2483          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2484          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2485          "mr 11,%1\n\t"                                           \
2486          "mr %0,3\n\t"                                            \
2487          "ld 2,-16(11)" /* restore tocptr */                      \
2488          : /*out*/   "=r" (_res)                                  \
2489          : /*in*/    "r" (&_argvec[2])                            \
2490          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2491       );                                                          \
2492       lval = (__typeof__(lval)) _res;                             \
2493    } while (0)
2494
2495 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2496                                  arg7,arg8,arg9)                  \
2497    do {                                                           \
2498       volatile OrigFn        _orig = (orig);                      \
2499       volatile unsigned long _argvec[3+9];                        \
2500       volatile unsigned long _res;                                \
2501       /* _argvec[0] holds current r2 across the call */           \
2502       _argvec[1]   = (unsigned long)_orig.r2;                     \
2503       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2504       _argvec[2+1] = (unsigned long)arg1;                         \
2505       _argvec[2+2] = (unsigned long)arg2;                         \
2506       _argvec[2+3] = (unsigned long)arg3;                         \
2507       _argvec[2+4] = (unsigned long)arg4;                         \
2508       _argvec[2+5] = (unsigned long)arg5;                         \
2509       _argvec[2+6] = (unsigned long)arg6;                         \
2510       _argvec[2+7] = (unsigned long)arg7;                         \
2511       _argvec[2+8] = (unsigned long)arg8;                         \
2512       _argvec[2+9] = (unsigned long)arg9;                         \
2513       __asm__ volatile(                                           \
2514          "mr 11,%1\n\t"                                           \
2515          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2516          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2517          "addi 1,1,-128\n\t"  /* expand stack frame */            \
2518          /* arg9 */                                               \
2519          "ld  3,72(11)\n\t"                                       \
2520          "std 3,112(1)\n\t"                                       \
2521          /* args1-8 */                                            \
2522          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2523          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2524          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2525          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2526          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2527          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2528          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2529          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2530          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2531          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2532          "mr 11,%1\n\t"                                           \
2533          "mr %0,3\n\t"                                            \
2534          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2535          "addi 1,1,128"     /* restore frame */                   \
2536          : /*out*/   "=r" (_res)                                  \
2537          : /*in*/    "r" (&_argvec[2])                            \
2538          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2539       );                                                          \
2540       lval = (__typeof__(lval)) _res;                             \
2541    } while (0)
2542
2543 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2544                                   arg7,arg8,arg9,arg10)           \
2545    do {                                                           \
2546       volatile OrigFn        _orig = (orig);                      \
2547       volatile unsigned long _argvec[3+10];                       \
2548       volatile unsigned long _res;                                \
2549       /* _argvec[0] holds current r2 across the call */           \
2550       _argvec[1]   = (unsigned long)_orig.r2;                     \
2551       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2552       _argvec[2+1] = (unsigned long)arg1;                         \
2553       _argvec[2+2] = (unsigned long)arg2;                         \
2554       _argvec[2+3] = (unsigned long)arg3;                         \
2555       _argvec[2+4] = (unsigned long)arg4;                         \
2556       _argvec[2+5] = (unsigned long)arg5;                         \
2557       _argvec[2+6] = (unsigned long)arg6;                         \
2558       _argvec[2+7] = (unsigned long)arg7;                         \
2559       _argvec[2+8] = (unsigned long)arg8;                         \
2560       _argvec[2+9] = (unsigned long)arg9;                         \
2561       _argvec[2+10] = (unsigned long)arg10;                       \
2562       __asm__ volatile(                                           \
2563          "mr 11,%1\n\t"                                           \
2564          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2565          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2566          "addi 1,1,-128\n\t"  /* expand stack frame */            \
2567          /* arg10 */                                              \
2568          "ld  3,80(11)\n\t"                                       \
2569          "std 3,120(1)\n\t"                                       \
2570          /* arg9 */                                               \
2571          "ld  3,72(11)\n\t"                                       \
2572          "std 3,112(1)\n\t"                                       \
2573          /* args1-8 */                                            \
2574          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2575          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2576          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2577          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2578          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2579          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2580          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2581          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2582          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2583          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2584          "mr 11,%1\n\t"                                           \
2585          "mr %0,3\n\t"                                            \
2586          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2587          "addi 1,1,128"     /* restore frame */                   \
2588          : /*out*/   "=r" (_res)                                  \
2589          : /*in*/    "r" (&_argvec[2])                            \
2590          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2591       );                                                          \
2592       lval = (__typeof__(lval)) _res;                             \
2593    } while (0)
2594
2595 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2596                                   arg7,arg8,arg9,arg10,arg11)     \
2597    do {                                                           \
2598       volatile OrigFn        _orig = (orig);                      \
2599       volatile unsigned long _argvec[3+11];                       \
2600       volatile unsigned long _res;                                \
2601       /* _argvec[0] holds current r2 across the call */           \
2602       _argvec[1]   = (unsigned long)_orig.r2;                     \
2603       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2604       _argvec[2+1] = (unsigned long)arg1;                         \
2605       _argvec[2+2] = (unsigned long)arg2;                         \
2606       _argvec[2+3] = (unsigned long)arg3;                         \
2607       _argvec[2+4] = (unsigned long)arg4;                         \
2608       _argvec[2+5] = (unsigned long)arg5;                         \
2609       _argvec[2+6] = (unsigned long)arg6;                         \
2610       _argvec[2+7] = (unsigned long)arg7;                         \
2611       _argvec[2+8] = (unsigned long)arg8;                         \
2612       _argvec[2+9] = (unsigned long)arg9;                         \
2613       _argvec[2+10] = (unsigned long)arg10;                       \
2614       _argvec[2+11] = (unsigned long)arg11;                       \
2615       __asm__ volatile(                                           \
2616          "mr 11,%1\n\t"                                           \
2617          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2618          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2619          "addi 1,1,-144\n\t"  /* expand stack frame */            \
2620          /* arg11 */                                              \
2621          "ld  3,88(11)\n\t"                                       \
2622          "std 3,128(1)\n\t"                                       \
2623          /* arg10 */                                              \
2624          "ld  3,80(11)\n\t"                                       \
2625          "std 3,120(1)\n\t"                                       \
2626          /* arg9 */                                               \
2627          "ld  3,72(11)\n\t"                                       \
2628          "std 3,112(1)\n\t"                                       \
2629          /* args1-8 */                                            \
2630          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2631          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2632          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2633          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2634          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2635          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2636          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2637          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2638          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2639          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2640          "mr 11,%1\n\t"                                           \
2641          "mr %0,3\n\t"                                            \
2642          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2643          "addi 1,1,144"     /* restore frame */                   \
2644          : /*out*/   "=r" (_res)                                  \
2645          : /*in*/    "r" (&_argvec[2])                            \
2646          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2647       );                                                          \
2648       lval = (__typeof__(lval)) _res;                             \
2649    } while (0)
2650
2651 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2652                                 arg7,arg8,arg9,arg10,arg11,arg12) \
2653    do {                                                           \
2654       volatile OrigFn        _orig = (orig);                      \
2655       volatile unsigned long _argvec[3+12];                       \
2656       volatile unsigned long _res;                                \
2657       /* _argvec[0] holds current r2 across the call */           \
2658       _argvec[1]   = (unsigned long)_orig.r2;                     \
2659       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2660       _argvec[2+1] = (unsigned long)arg1;                         \
2661       _argvec[2+2] = (unsigned long)arg2;                         \
2662       _argvec[2+3] = (unsigned long)arg3;                         \
2663       _argvec[2+4] = (unsigned long)arg4;                         \
2664       _argvec[2+5] = (unsigned long)arg5;                         \
2665       _argvec[2+6] = (unsigned long)arg6;                         \
2666       _argvec[2+7] = (unsigned long)arg7;                         \
2667       _argvec[2+8] = (unsigned long)arg8;                         \
2668       _argvec[2+9] = (unsigned long)arg9;                         \
2669       _argvec[2+10] = (unsigned long)arg10;                       \
2670       _argvec[2+11] = (unsigned long)arg11;                       \
2671       _argvec[2+12] = (unsigned long)arg12;                       \
2672       __asm__ volatile(                                           \
2673          "mr 11,%1\n\t"                                           \
2674          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2675          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2676          "addi 1,1,-144\n\t"  /* expand stack frame */            \
2677          /* arg12 */                                              \
2678          "ld  3,96(11)\n\t"                                       \
2679          "std 3,136(1)\n\t"                                       \
2680          /* arg11 */                                              \
2681          "ld  3,88(11)\n\t"                                       \
2682          "std 3,128(1)\n\t"                                       \
2683          /* arg10 */                                              \
2684          "ld  3,80(11)\n\t"                                       \
2685          "std 3,120(1)\n\t"                                       \
2686          /* arg9 */                                               \
2687          "ld  3,72(11)\n\t"                                       \
2688          "std 3,112(1)\n\t"                                       \
2689          /* args1-8 */                                            \
2690          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2691          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2692          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2693          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2694          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2695          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2696          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2697          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2698          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2699          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2700          "mr 11,%1\n\t"                                           \
2701          "mr %0,3\n\t"                                            \
2702          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2703          "addi 1,1,144"     /* restore frame */                   \
2704          : /*out*/   "=r" (_res)                                  \
2705          : /*in*/    "r" (&_argvec[2])                            \
2706          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2707       );                                                          \
2708       lval = (__typeof__(lval)) _res;                             \
2709    } while (0)
2710
2711 #endif /* PLAT_ppc64_linux */
2712
2713 /* ------------------------- arm-linux ------------------------- */
2714
2715 #if defined(PLAT_arm_linux)
2716
2717 /* These regs are trashed by the hidden call. */
2718 #define __CALLER_SAVED_REGS "r0", "r1", "r2", "r3","r4","r14"
2719
2720 /* These CALL_FN_ macros assume that on arm-linux, sizeof(unsigned
2721    long) == 4. */
2722
2723 #define CALL_FN_W_v(lval, orig)                                   \
2724    do {                                                           \
2725       volatile OrigFn        _orig = (orig);                      \
2726       volatile unsigned long _argvec[1];                          \
2727       volatile unsigned long _res;                                \
2728       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2729       __asm__ volatile(                                           \
2730          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2731          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2732          "mov %0, r0\n"                                           \
2733          : /*out*/   "=r" (_res)                                  \
2734          : /*in*/    "0" (&_argvec[0])                            \
2735          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2736       );                                                          \
2737       lval = (__typeof__(lval)) _res;                             \
2738    } while (0)
2739
2740 #define CALL_FN_W_W(lval, orig, arg1)                             \
2741    do {                                                           \
2742       volatile OrigFn        _orig = (orig);                      \
2743       volatile unsigned long _argvec[2];                          \
2744       volatile unsigned long _res;                                \
2745       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2746       _argvec[1] = (unsigned long)(arg1);                         \
2747       __asm__ volatile(                                           \
2748          "ldr r0, [%1, #4] \n\t"                                  \
2749          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2750          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2751          "mov %0, r0\n"                                           \
2752          : /*out*/   "=r" (_res)                                  \
2753          : /*in*/    "0" (&_argvec[0])                            \
2754          : /*trash*/ "cc", "memory",  __CALLER_SAVED_REGS         \
2755       );                                                          \
2756       lval = (__typeof__(lval)) _res;                             \
2757    } while (0)
2758
2759 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2760    do {                                                           \
2761       volatile OrigFn        _orig = (orig);                      \
2762       volatile unsigned long _argvec[3];                          \
2763       volatile unsigned long _res;                                \
2764       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2765       _argvec[1] = (unsigned long)(arg1);                         \
2766       _argvec[2] = (unsigned long)(arg2);                         \
2767       __asm__ volatile(                                           \
2768          "ldr r0, [%1, #4] \n\t"                                  \
2769          "ldr r1, [%1, #8] \n\t"                                  \
2770          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2771          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2772          "mov %0, r0\n"                                           \
2773          : /*out*/   "=r" (_res)                                  \
2774          : /*in*/    "0" (&_argvec[0])                            \
2775          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2776       );                                                          \
2777       lval = (__typeof__(lval)) _res;                             \
2778    } while (0)
2779
2780 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2781    do {                                                           \
2782       volatile OrigFn        _orig = (orig);                      \
2783       volatile unsigned long _argvec[4];                          \
2784       volatile unsigned long _res;                                \
2785       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2786       _argvec[1] = (unsigned long)(arg1);                         \
2787       _argvec[2] = (unsigned long)(arg2);                         \
2788       _argvec[3] = (unsigned long)(arg3);                         \
2789       __asm__ volatile(                                           \
2790          "ldr r0, [%1, #4] \n\t"                                  \
2791          "ldr r1, [%1, #8] \n\t"                                  \
2792          "ldr r2, [%1, #12] \n\t"                                 \
2793          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2794          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2795          "mov %0, r0\n"                                           \
2796          : /*out*/   "=r" (_res)                                  \
2797          : /*in*/    "0" (&_argvec[0])                            \
2798          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2799       );                                                          \
2800       lval = (__typeof__(lval)) _res;                             \
2801    } while (0)
2802
2803 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2804    do {                                                           \
2805       volatile OrigFn        _orig = (orig);                      \
2806       volatile unsigned long _argvec[5];                          \
2807       volatile unsigned long _res;                                \
2808       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2809       _argvec[1] = (unsigned long)(arg1);                         \
2810       _argvec[2] = (unsigned long)(arg2);                         \
2811       _argvec[3] = (unsigned long)(arg3);                         \
2812       _argvec[4] = (unsigned long)(arg4);                         \
2813       __asm__ volatile(                                           \
2814          "ldr r0, [%1, #4] \n\t"                                  \
2815          "ldr r1, [%1, #8] \n\t"                                  \
2816          "ldr r2, [%1, #12] \n\t"                                 \
2817          "ldr r3, [%1, #16] \n\t"                                 \
2818          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2819          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2820          "mov %0, r0"                                             \
2821          : /*out*/   "=r" (_res)                                  \
2822          : /*in*/    "0" (&_argvec[0])                            \
2823          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2824       );                                                          \
2825       lval = (__typeof__(lval)) _res;                             \
2826    } while (0)
2827
2828 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2829    do {                                                           \
2830       volatile OrigFn        _orig = (orig);                      \
2831       volatile unsigned long _argvec[6];                          \
2832       volatile unsigned long _res;                                \
2833       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2834       _argvec[1] = (unsigned long)(arg1);                         \
2835       _argvec[2] = (unsigned long)(arg2);                         \
2836       _argvec[3] = (unsigned long)(arg3);                         \
2837       _argvec[4] = (unsigned long)(arg4);                         \
2838       _argvec[5] = (unsigned long)(arg5);                         \
2839       __asm__ volatile(                                           \
2840          "ldr r0, [%1, #20] \n\t"                                 \
2841          "push {r0} \n\t"                                         \
2842          "ldr r0, [%1, #4] \n\t"                                  \
2843          "ldr r1, [%1, #8] \n\t"                                  \
2844          "ldr r2, [%1, #12] \n\t"                                 \
2845          "ldr r3, [%1, #16] \n\t"                                 \
2846          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2847          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2848          "add sp, sp, #4 \n\t"                                    \
2849          "mov %0, r0"                                             \
2850          : /*out*/   "=r" (_res)                                  \
2851          : /*in*/    "0" (&_argvec[0])                            \
2852          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2853       );                                                          \
2854       lval = (__typeof__(lval)) _res;                             \
2855    } while (0)
2856
2857 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2858    do {                                                           \
2859       volatile OrigFn        _orig = (orig);                      \
2860       volatile unsigned long _argvec[7];                          \
2861       volatile unsigned long _res;                                \
2862       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2863       _argvec[1] = (unsigned long)(arg1);                         \
2864       _argvec[2] = (unsigned long)(arg2);                         \
2865       _argvec[3] = (unsigned long)(arg3);                         \
2866       _argvec[4] = (unsigned long)(arg4);                         \
2867       _argvec[5] = (unsigned long)(arg5);                         \
2868       _argvec[6] = (unsigned long)(arg6);                         \
2869       __asm__ volatile(                                           \
2870          "ldr r0, [%1, #20] \n\t"                                 \
2871          "ldr r1, [%1, #24] \n\t"                                 \
2872          "push {r0, r1} \n\t"                                     \
2873          "ldr r0, [%1, #4] \n\t"                                  \
2874          "ldr r1, [%1, #8] \n\t"                                  \
2875          "ldr r2, [%1, #12] \n\t"                                 \
2876          "ldr r3, [%1, #16] \n\t"                                 \
2877          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2878          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2879          "add sp, sp, #8 \n\t"                                    \
2880          "mov %0, r0"                                             \
2881          : /*out*/   "=r" (_res)                                  \
2882          : /*in*/    "0" (&_argvec[0])                            \
2883          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2884       );                                                          \
2885       lval = (__typeof__(lval)) _res;                             \
2886    } while (0)
2887
2888 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2889                                  arg7)                            \
2890    do {                                                           \
2891       volatile OrigFn        _orig = (orig);                      \
2892       volatile unsigned long _argvec[8];                          \
2893       volatile unsigned long _res;                                \
2894       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2895       _argvec[1] = (unsigned long)(arg1);                         \
2896       _argvec[2] = (unsigned long)(arg2);                         \
2897       _argvec[3] = (unsigned long)(arg3);                         \
2898       _argvec[4] = (unsigned long)(arg4);                         \
2899       _argvec[5] = (unsigned long)(arg5);                         \
2900       _argvec[6] = (unsigned long)(arg6);                         \
2901       _argvec[7] = (unsigned long)(arg7);                         \
2902       __asm__ volatile(                                           \
2903          "ldr r0, [%1, #20] \n\t"                                 \
2904          "ldr r1, [%1, #24] \n\t"                                 \
2905          "ldr r2, [%1, #28] \n\t"                                 \
2906          "push {r0, r1, r2} \n\t"                                 \
2907          "ldr r0, [%1, #4] \n\t"                                  \
2908          "ldr r1, [%1, #8] \n\t"                                  \
2909          "ldr r2, [%1, #12] \n\t"                                 \
2910          "ldr r3, [%1, #16] \n\t"                                 \
2911          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2912          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2913          "add sp, sp, #12 \n\t"                                   \
2914          "mov %0, r0"                                             \
2915          : /*out*/   "=r" (_res)                                  \
2916          : /*in*/    "0" (&_argvec[0])                            \
2917          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2918       );                                                          \
2919       lval = (__typeof__(lval)) _res;                             \
2920    } while (0)
2921
2922 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2923                                  arg7,arg8)                       \
2924    do {                                                           \
2925       volatile OrigFn        _orig = (orig);                      \
2926       volatile unsigned long _argvec[9];                          \
2927       volatile unsigned long _res;                                \
2928       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2929       _argvec[1] = (unsigned long)(arg1);                         \
2930       _argvec[2] = (unsigned long)(arg2);                         \
2931       _argvec[3] = (unsigned long)(arg3);                         \
2932       _argvec[4] = (unsigned long)(arg4);                         \
2933       _argvec[5] = (unsigned long)(arg5);                         \
2934       _argvec[6] = (unsigned long)(arg6);                         \
2935       _argvec[7] = (unsigned long)(arg7);                         \
2936       _argvec[8] = (unsigned long)(arg8);                         \
2937       __asm__ volatile(                                           \
2938          "ldr r0, [%1, #20] \n\t"                                 \
2939          "ldr r1, [%1, #24] \n\t"                                 \
2940          "ldr r2, [%1, #28] \n\t"                                 \
2941          "ldr r3, [%1, #32] \n\t"                                 \
2942          "push {r0, r1, r2, r3} \n\t"                             \
2943          "ldr r0, [%1, #4] \n\t"                                  \
2944          "ldr r1, [%1, #8] \n\t"                                  \
2945          "ldr r2, [%1, #12] \n\t"                                 \
2946          "ldr r3, [%1, #16] \n\t"                                 \
2947          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2948          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2949          "add sp, sp, #16 \n\t"                                   \
2950          "mov %0, r0"                                             \
2951          : /*out*/   "=r" (_res)                                  \
2952          : /*in*/    "0" (&_argvec[0])                            \
2953          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2954       );                                                          \
2955       lval = (__typeof__(lval)) _res;                             \
2956    } while (0)
2957
2958 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2959                                  arg7,arg8,arg9)                  \
2960    do {                                                           \
2961       volatile OrigFn        _orig = (orig);                      \
2962       volatile unsigned long _argvec[10];                         \
2963       volatile unsigned long _res;                                \
2964       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2965       _argvec[1] = (unsigned long)(arg1);                         \
2966       _argvec[2] = (unsigned long)(arg2);                         \
2967       _argvec[3] = (unsigned long)(arg3);                         \
2968       _argvec[4] = (unsigned long)(arg4);                         \
2969       _argvec[5] = (unsigned long)(arg5);                         \
2970       _argvec[6] = (unsigned long)(arg6);                         \
2971       _argvec[7] = (unsigned long)(arg7);                         \
2972       _argvec[8] = (unsigned long)(arg8);                         \
2973       _argvec[9] = (unsigned long)(arg9);                         \
2974       __asm__ volatile(                                           \
2975          "ldr r0, [%1, #20] \n\t"                                 \
2976          "ldr r1, [%1, #24] \n\t"                                 \
2977          "ldr r2, [%1, #28] \n\t"                                 \
2978          "ldr r3, [%1, #32] \n\t"                                 \
2979          "ldr r4, [%1, #36] \n\t"                                 \
2980          "push {r0, r1, r2, r3, r4} \n\t"                         \
2981          "ldr r0, [%1, #4] \n\t"                                  \
2982          "ldr r1, [%1, #8] \n\t"                                  \
2983          "ldr r2, [%1, #12] \n\t"                                 \
2984          "ldr r3, [%1, #16] \n\t"                                 \
2985          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2986          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2987          "add sp, sp, #20 \n\t"                                   \
2988          "mov %0, r0"                                             \
2989          : /*out*/   "=r" (_res)                                  \
2990          : /*in*/    "0" (&_argvec[0])                            \
2991          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2992       );                                                          \
2993       lval = (__typeof__(lval)) _res;                             \
2994    } while (0)
2995
2996 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2997                                   arg7,arg8,arg9,arg10)           \
2998    do {                                                           \
2999       volatile OrigFn        _orig = (orig);                      \
3000       volatile unsigned long _argvec[11];                         \
3001       volatile unsigned long _res;                                \
3002       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3003       _argvec[1] = (unsigned long)(arg1);                         \
3004       _argvec[2] = (unsigned long)(arg2);                         \
3005       _argvec[3] = (unsigned long)(arg3);                         \
3006       _argvec[4] = (unsigned long)(arg4);                         \
3007       _argvec[5] = (unsigned long)(arg5);                         \
3008       _argvec[6] = (unsigned long)(arg6);                         \
3009       _argvec[7] = (unsigned long)(arg7);                         \
3010       _argvec[8] = (unsigned long)(arg8);                         \
3011       _argvec[9] = (unsigned long)(arg9);                         \
3012       _argvec[10] = (unsigned long)(arg10);                       \
3013       __asm__ volatile(                                           \
3014          "ldr r0, [%1, #40] \n\t"                                 \
3015          "push {r0} \n\t"                                         \
3016          "ldr r0, [%1, #20] \n\t"                                 \
3017          "ldr r1, [%1, #24] \n\t"                                 \
3018          "ldr r2, [%1, #28] \n\t"                                 \
3019          "ldr r3, [%1, #32] \n\t"                                 \
3020          "ldr r4, [%1, #36] \n\t"                                 \
3021          "push {r0, r1, r2, r3, r4} \n\t"                         \
3022          "ldr r0, [%1, #4] \n\t"                                  \
3023          "ldr r1, [%1, #8] \n\t"                                  \
3024          "ldr r2, [%1, #12] \n\t"                                 \
3025          "ldr r3, [%1, #16] \n\t"                                 \
3026          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3027          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3028          "add sp, sp, #24 \n\t"                                   \
3029          "mov %0, r0"                                             \
3030          : /*out*/   "=r" (_res)                                  \
3031          : /*in*/    "0" (&_argvec[0])                            \
3032          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3033       );                                                          \
3034       lval = (__typeof__(lval)) _res;                             \
3035    } while (0)
3036
3037 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
3038                                   arg6,arg7,arg8,arg9,arg10,      \
3039                                   arg11)                          \
3040    do {                                                           \
3041       volatile OrigFn        _orig = (orig);                      \
3042       volatile unsigned long _argvec[12];                         \
3043       volatile unsigned long _res;                                \
3044       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3045       _argvec[1] = (unsigned long)(arg1);                         \
3046       _argvec[2] = (unsigned long)(arg2);                         \
3047       _argvec[3] = (unsigned long)(arg3);                         \
3048       _argvec[4] = (unsigned long)(arg4);                         \
3049       _argvec[5] = (unsigned long)(arg5);                         \
3050       _argvec[6] = (unsigned long)(arg6);                         \
3051       _argvec[7] = (unsigned long)(arg7);                         \
3052       _argvec[8] = (unsigned long)(arg8);                         \
3053       _argvec[9] = (unsigned long)(arg9);                         \
3054       _argvec[10] = (unsigned long)(arg10);                       \
3055       _argvec[11] = (unsigned long)(arg11);                       \
3056       __asm__ volatile(                                           \
3057          "ldr r0, [%1, #40] \n\t"                                 \
3058          "ldr r1, [%1, #44] \n\t"                                 \
3059          "push {r0, r1} \n\t"                                     \
3060          "ldr r0, [%1, #20] \n\t"                                 \
3061          "ldr r1, [%1, #24] \n\t"                                 \
3062          "ldr r2, [%1, #28] \n\t"                                 \
3063          "ldr r3, [%1, #32] \n\t"                                 \
3064          "ldr r4, [%1, #36] \n\t"                                 \
3065          "push {r0, r1, r2, r3, r4} \n\t"                         \
3066          "ldr r0, [%1, #4] \n\t"                                  \
3067          "ldr r1, [%1, #8] \n\t"                                  \
3068          "ldr r2, [%1, #12] \n\t"                                 \
3069          "ldr r3, [%1, #16] \n\t"                                 \
3070          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3071          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3072          "add sp, sp, #28 \n\t"                                   \
3073          "mov %0, r0"                                             \
3074          : /*out*/   "=r" (_res)                                  \
3075          : /*in*/    "0" (&_argvec[0])                            \
3076          : /*trash*/ "cc", "memory",__CALLER_SAVED_REGS           \
3077       );                                                          \
3078       lval = (__typeof__(lval)) _res;                             \
3079    } while (0)
3080
3081 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
3082                                   arg6,arg7,arg8,arg9,arg10,      \
3083                                   arg11,arg12)                    \
3084    do {                                                           \
3085       volatile OrigFn        _orig = (orig);                      \
3086       volatile unsigned long _argvec[13];                         \
3087       volatile unsigned long _res;                                \
3088       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3089       _argvec[1] = (unsigned long)(arg1);                         \
3090       _argvec[2] = (unsigned long)(arg2);                         \
3091       _argvec[3] = (unsigned long)(arg3);                         \
3092       _argvec[4] = (unsigned long)(arg4);                         \
3093       _argvec[5] = (unsigned long)(arg5);                         \
3094       _argvec[6] = (unsigned long)(arg6);                         \
3095       _argvec[7] = (unsigned long)(arg7);                         \
3096       _argvec[8] = (unsigned long)(arg8);                         \
3097       _argvec[9] = (unsigned long)(arg9);                         \
3098       _argvec[10] = (unsigned long)(arg10);                       \
3099       _argvec[11] = (unsigned long)(arg11);                       \
3100       _argvec[12] = (unsigned long)(arg12);                       \
3101       __asm__ volatile(                                           \
3102          "ldr r0, [%1, #40] \n\t"                                 \
3103          "ldr r1, [%1, #44] \n\t"                                 \
3104          "ldr r2, [%1, #48] \n\t"                                 \
3105          "push {r0, r1, r2} \n\t"                                 \
3106          "ldr r0, [%1, #20] \n\t"                                 \
3107          "ldr r1, [%1, #24] \n\t"                                 \
3108          "ldr r2, [%1, #28] \n\t"                                 \
3109          "ldr r3, [%1, #32] \n\t"                                 \
3110          "ldr r4, [%1, #36] \n\t"                                 \
3111          "push {r0, r1, r2, r3, r4} \n\t"                         \
3112          "ldr r0, [%1, #4] \n\t"                                  \
3113          "ldr r1, [%1, #8] \n\t"                                  \
3114          "ldr r2, [%1, #12] \n\t"                                 \
3115          "ldr r3, [%1, #16] \n\t"                                 \
3116          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3117          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3118          "add sp, sp, #32 \n\t"                                   \
3119          "mov %0, r0"                                             \
3120          : /*out*/   "=r" (_res)                                  \
3121          : /*in*/    "0" (&_argvec[0])                            \
3122          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3123       );                                                          \
3124       lval = (__typeof__(lval)) _res;                             \
3125    } while (0)
3126
3127 #endif /* PLAT_arm_linux */
3128
3129 /* ------------------------ ppc32-aix5 ------------------------- */
3130
3131 #if defined(PLAT_ppc32_aix5)
3132
3133 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
3134
3135 /* These regs are trashed by the hidden call. */
3136 #define __CALLER_SAVED_REGS                                       \
3137    "lr", "ctr", "xer",                                            \
3138    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
3139    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
3140    "r11", "r12", "r13"
3141
3142 /* Expand the stack frame, copying enough info that unwinding
3143    still works.  Trashes r3. */
3144
3145 #define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr)                      \
3146          "addi 1,1,-" #_n_fr "\n\t"                               \
3147          "lwz  3," #_n_fr "(1)\n\t"                               \
3148          "stw  3,0(1)\n\t"
3149
3150 #define VG_CONTRACT_FRAME_BY(_n_fr)                               \
3151          "addi 1,1," #_n_fr "\n\t"
3152
3153 /* These CALL_FN_ macros assume that on ppc32-aix5, sizeof(unsigned
3154    long) == 4. */
3155
3156 #define CALL_FN_W_v(lval, orig)                                   \
3157    do {                                                           \
3158       volatile OrigFn        _orig = (orig);                      \
3159       volatile unsigned long _argvec[3+0];                        \
3160       volatile unsigned long _res;                                \
3161       /* _argvec[0] holds current r2 across the call */           \
3162       _argvec[1] = (unsigned long)_orig.r2;                       \
3163       _argvec[2] = (unsigned long)_orig.nraddr;                   \
3164       __asm__ volatile(                                           \
3165          "mr 11,%1\n\t"                                           \
3166          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3167          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3168          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3169          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3170          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3171          "mr 11,%1\n\t"                                           \
3172          "mr %0,3\n\t"                                            \
3173          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3174          VG_CONTRACT_FRAME_BY(512)                                \
3175          : /*out*/   "=r" (_res)                                  \
3176          : /*in*/    "r" (&_argvec[2])                            \
3177          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3178       );                                                          \
3179       lval = (__typeof__(lval)) _res;                             \
3180    } while (0)
3181
3182 #define CALL_FN_W_W(lval, orig, arg1)                             \
3183    do {                                                           \
3184       volatile OrigFn        _orig = (orig);                      \
3185       volatile unsigned long _argvec[3+1];                        \
3186       volatile unsigned long _res;                                \
3187       /* _argvec[0] holds current r2 across the call */           \
3188       _argvec[1]   = (unsigned long)_orig.r2;                     \
3189       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3190       _argvec[2+1] = (unsigned long)arg1;                         \
3191       __asm__ volatile(                                           \
3192          "mr 11,%1\n\t"                                           \
3193          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3194          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3195          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3196          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3197          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3198          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3199          "mr 11,%1\n\t"                                           \
3200          "mr %0,3\n\t"                                            \
3201          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3202          VG_CONTRACT_FRAME_BY(512)                                \
3203          : /*out*/   "=r" (_res)                                  \
3204          : /*in*/    "r" (&_argvec[2])                            \
3205          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3206       );                                                          \
3207       lval = (__typeof__(lval)) _res;                             \
3208    } while (0)
3209
3210 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3211    do {                                                           \
3212       volatile OrigFn        _orig = (orig);                      \
3213       volatile unsigned long _argvec[3+2];                        \
3214       volatile unsigned long _res;                                \
3215       /* _argvec[0] holds current r2 across the call */           \
3216       _argvec[1]   = (unsigned long)_orig.r2;                     \
3217       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3218       _argvec[2+1] = (unsigned long)arg1;                         \
3219       _argvec[2+2] = (unsigned long)arg2;                         \
3220       __asm__ volatile(                                           \
3221          "mr 11,%1\n\t"                                           \
3222          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3223          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3224          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3225          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3226          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3227          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3228          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3229          "mr 11,%1\n\t"                                           \
3230          "mr %0,3\n\t"                                            \
3231          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3232          VG_CONTRACT_FRAME_BY(512)                                \
3233          : /*out*/   "=r" (_res)                                  \
3234          : /*in*/    "r" (&_argvec[2])                            \
3235          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3236       );                                                          \
3237       lval = (__typeof__(lval)) _res;                             \
3238    } while (0)
3239
3240 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3241    do {                                                           \
3242       volatile OrigFn        _orig = (orig);                      \
3243       volatile unsigned long _argvec[3+3];                        \
3244       volatile unsigned long _res;                                \
3245       /* _argvec[0] holds current r2 across the call */           \
3246       _argvec[1]   = (unsigned long)_orig.r2;                     \
3247       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3248       _argvec[2+1] = (unsigned long)arg1;                         \
3249       _argvec[2+2] = (unsigned long)arg2;                         \
3250       _argvec[2+3] = (unsigned long)arg3;                         \
3251       __asm__ volatile(                                           \
3252          "mr 11,%1\n\t"                                           \
3253          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3254          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3255          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3256          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3257          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3258          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3259          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3260          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3261          "mr 11,%1\n\t"                                           \
3262          "mr %0,3\n\t"                                            \
3263          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3264          VG_CONTRACT_FRAME_BY(512)                                \
3265          : /*out*/   "=r" (_res)                                  \
3266          : /*in*/    "r" (&_argvec[2])                            \
3267          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3268       );                                                          \
3269       lval = (__typeof__(lval)) _res;                             \
3270    } while (0)
3271
3272 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3273    do {                                                           \
3274       volatile OrigFn        _orig = (orig);                      \
3275       volatile unsigned long _argvec[3+4];                        \
3276       volatile unsigned long _res;                                \
3277       /* _argvec[0] holds current r2 across the call */           \
3278       _argvec[1]   = (unsigned long)_orig.r2;                     \
3279       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3280       _argvec[2+1] = (unsigned long)arg1;                         \
3281       _argvec[2+2] = (unsigned long)arg2;                         \
3282       _argvec[2+3] = (unsigned long)arg3;                         \
3283       _argvec[2+4] = (unsigned long)arg4;                         \
3284       __asm__ volatile(                                           \
3285          "mr 11,%1\n\t"                                           \
3286          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3287          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3288          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3289          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3290          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3291          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3292          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3293          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3294          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3295          "mr 11,%1\n\t"                                           \
3296          "mr %0,3\n\t"                                            \
3297          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3298          VG_CONTRACT_FRAME_BY(512)                                \
3299          : /*out*/   "=r" (_res)                                  \
3300          : /*in*/    "r" (&_argvec[2])                            \
3301          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3302       );                                                          \
3303       lval = (__typeof__(lval)) _res;                             \
3304    } while (0)
3305
3306 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3307    do {                                                           \
3308       volatile OrigFn        _orig = (orig);                      \
3309       volatile unsigned long _argvec[3+5];                        \
3310       volatile unsigned long _res;                                \
3311       /* _argvec[0] holds current r2 across the call */           \
3312       _argvec[1]   = (unsigned long)_orig.r2;                     \
3313       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3314       _argvec[2+1] = (unsigned long)arg1;                         \
3315       _argvec[2+2] = (unsigned long)arg2;                         \
3316       _argvec[2+3] = (unsigned long)arg3;                         \
3317       _argvec[2+4] = (unsigned long)arg4;                         \
3318       _argvec[2+5] = (unsigned long)arg5;                         \
3319       __asm__ volatile(                                           \
3320          "mr 11,%1\n\t"                                           \
3321          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3322          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3323          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3324          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3325          "lwz  4, 8(11)\n\t" /* arg2->r4 */                       \
3326          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3327          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3328          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3329          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3330          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3331          "mr 11,%1\n\t"                                           \
3332          "mr %0,3\n\t"                                            \
3333          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3334          VG_CONTRACT_FRAME_BY(512)                                \
3335          : /*out*/   "=r" (_res)                                  \
3336          : /*in*/    "r" (&_argvec[2])                            \
3337          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3338       );                                                          \
3339       lval = (__typeof__(lval)) _res;                             \
3340    } while (0)
3341
3342 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3343    do {                                                           \
3344       volatile OrigFn        _orig = (orig);                      \
3345       volatile unsigned long _argvec[3+6];                        \
3346       volatile unsigned long _res;                                \
3347       /* _argvec[0] holds current r2 across the call */           \
3348       _argvec[1]   = (unsigned long)_orig.r2;                     \
3349       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3350       _argvec[2+1] = (unsigned long)arg1;                         \
3351       _argvec[2+2] = (unsigned long)arg2;                         \
3352       _argvec[2+3] = (unsigned long)arg3;                         \
3353       _argvec[2+4] = (unsigned long)arg4;                         \
3354       _argvec[2+5] = (unsigned long)arg5;                         \
3355       _argvec[2+6] = (unsigned long)arg6;                         \
3356       __asm__ volatile(                                           \
3357          "mr 11,%1\n\t"                                           \
3358          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3359          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3360          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3361          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3362          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3363          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3364          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3365          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3366          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3367          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3368          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3369          "mr 11,%1\n\t"                                           \
3370          "mr %0,3\n\t"                                            \
3371          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3372          VG_CONTRACT_FRAME_BY(512)                                \
3373          : /*out*/   "=r" (_res)                                  \
3374          : /*in*/    "r" (&_argvec[2])                            \
3375          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3376       );                                                          \
3377       lval = (__typeof__(lval)) _res;                             \
3378    } while (0)
3379
3380 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3381                                  arg7)                            \
3382    do {                                                           \
3383       volatile OrigFn        _orig = (orig);                      \
3384       volatile unsigned long _argvec[3+7];                        \
3385       volatile unsigned long _res;                                \
3386       /* _argvec[0] holds current r2 across the call */           \
3387       _argvec[1]   = (unsigned long)_orig.r2;                     \
3388       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3389       _argvec[2+1] = (unsigned long)arg1;                         \
3390       _argvec[2+2] = (unsigned long)arg2;                         \
3391       _argvec[2+3] = (unsigned long)arg3;                         \
3392       _argvec[2+4] = (unsigned long)arg4;                         \
3393       _argvec[2+5] = (unsigned long)arg5;                         \
3394       _argvec[2+6] = (unsigned long)arg6;                         \
3395       _argvec[2+7] = (unsigned long)arg7;                         \
3396       __asm__ volatile(                                           \
3397          "mr 11,%1\n\t"                                           \
3398          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3399          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3400          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3401          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3402          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3403          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3404          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3405          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3406          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3407          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3408          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3409          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3410          "mr 11,%1\n\t"                                           \
3411          "mr %0,3\n\t"                                            \
3412          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3413          VG_CONTRACT_FRAME_BY(512)                                \
3414          : /*out*/   "=r" (_res)                                  \
3415          : /*in*/    "r" (&_argvec[2])                            \
3416          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3417       );                                                          \
3418       lval = (__typeof__(lval)) _res;                             \
3419    } while (0)
3420
3421 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3422                                  arg7,arg8)                       \
3423    do {                                                           \
3424       volatile OrigFn        _orig = (orig);                      \
3425       volatile unsigned long _argvec[3+8];                        \
3426       volatile unsigned long _res;                                \
3427       /* _argvec[0] holds current r2 across the call */           \
3428       _argvec[1]   = (unsigned long)_orig.r2;                     \
3429       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3430       _argvec[2+1] = (unsigned long)arg1;                         \
3431       _argvec[2+2] = (unsigned long)arg2;                         \
3432       _argvec[2+3] = (unsigned long)arg3;                         \
3433       _argvec[2+4] = (unsigned long)arg4;                         \
3434       _argvec[2+5] = (unsigned long)arg5;                         \
3435       _argvec[2+6] = (unsigned long)arg6;                         \
3436       _argvec[2+7] = (unsigned long)arg7;                         \
3437       _argvec[2+8] = (unsigned long)arg8;                         \
3438       __asm__ volatile(                                           \
3439          "mr 11,%1\n\t"                                           \
3440          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3441          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3442          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3443          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3444          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3445          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3446          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3447          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3448          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3449          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3450          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3451          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3452          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3453          "mr 11,%1\n\t"                                           \
3454          "mr %0,3\n\t"                                            \
3455          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3456          VG_CONTRACT_FRAME_BY(512)                                \
3457          : /*out*/   "=r" (_res)                                  \
3458          : /*in*/    "r" (&_argvec[2])                            \
3459          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3460       );                                                          \
3461       lval = (__typeof__(lval)) _res;                             \
3462    } while (0)
3463
3464 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3465                                  arg7,arg8,arg9)                  \
3466    do {                                                           \
3467       volatile OrigFn        _orig = (orig);                      \
3468       volatile unsigned long _argvec[3+9];                        \
3469       volatile unsigned long _res;                                \
3470       /* _argvec[0] holds current r2 across the call */           \
3471       _argvec[1]   = (unsigned long)_orig.r2;                     \
3472       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3473       _argvec[2+1] = (unsigned long)arg1;                         \
3474       _argvec[2+2] = (unsigned long)arg2;                         \
3475       _argvec[2+3] = (unsigned long)arg3;                         \
3476       _argvec[2+4] = (unsigned long)arg4;                         \
3477       _argvec[2+5] = (unsigned long)arg5;                         \
3478       _argvec[2+6] = (unsigned long)arg6;                         \
3479       _argvec[2+7] = (unsigned long)arg7;                         \
3480       _argvec[2+8] = (unsigned long)arg8;                         \
3481       _argvec[2+9] = (unsigned long)arg9;                         \
3482       __asm__ volatile(                                           \
3483          "mr 11,%1\n\t"                                           \
3484          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3485          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3486          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3487          VG_EXPAND_FRAME_BY_trashes_r3(64)                        \
3488          /* arg9 */                                               \
3489          "lwz 3,36(11)\n\t"                                       \
3490          "stw 3,56(1)\n\t"                                        \
3491          /* args1-8 */                                            \
3492          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3493          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3494          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3495          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3496          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3497          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3498          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3499          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3500          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3501          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3502          "mr 11,%1\n\t"                                           \
3503          "mr %0,3\n\t"                                            \
3504          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3505          VG_CONTRACT_FRAME_BY(64)                                 \
3506          VG_CONTRACT_FRAME_BY(512)                                \
3507          : /*out*/   "=r" (_res)                                  \
3508          : /*in*/    "r" (&_argvec[2])                            \
3509          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3510       );                                                          \
3511       lval = (__typeof__(lval)) _res;                             \
3512    } while (0)
3513
3514 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3515                                   arg7,arg8,arg9,arg10)           \
3516    do {                                                           \
3517       volatile OrigFn        _orig = (orig);                      \
3518       volatile unsigned long _argvec[3+10];                       \
3519       volatile unsigned long _res;                                \
3520       /* _argvec[0] holds current r2 across the call */           \
3521       _argvec[1]   = (unsigned long)_orig.r2;                     \
3522       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3523       _argvec[2+1] = (unsigned long)arg1;                         \
3524       _argvec[2+2] = (unsigned long)arg2;                         \
3525       _argvec[2+3] = (unsigned long)arg3;                         \
3526       _argvec[2+4] = (unsigned long)arg4;                         \
3527       _argvec[2+5] = (unsigned long)arg5;                         \
3528       _argvec[2+6] = (unsigned long)arg6;                         \
3529       _argvec[2+7] = (unsigned long)arg7;                         \
3530       _argvec[2+8] = (unsigned long)arg8;                         \
3531       _argvec[2+9] = (unsigned long)arg9;                         \
3532       _argvec[2+10] = (unsigned long)arg10;                       \
3533       __asm__ volatile(                                           \
3534          "mr 11,%1\n\t"                                           \
3535          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3536          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3537          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3538          VG_EXPAND_FRAME_BY_trashes_r3(64)                        \
3539          /* arg10 */                                              \
3540          "lwz 3,40(11)\n\t"                                       \
3541          "stw 3,60(1)\n\t"                                        \
3542          /* arg9 */                                               \
3543          "lwz 3,36(11)\n\t"                                       \
3544          "stw 3,56(1)\n\t"                                        \
3545          /* args1-8 */                                            \
3546          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3547          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3548          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3549          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3550          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3551          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3552          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3553          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3554          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3555          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3556          "mr 11,%1\n\t"                                           \
3557          "mr %0,3\n\t"                                            \
3558          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3559          VG_CONTRACT_FRAME_BY(64)                                 \
3560          VG_CONTRACT_FRAME_BY(512)                                \
3561          : /*out*/   "=r" (_res)                                  \
3562          : /*in*/    "r" (&_argvec[2])                            \
3563          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3564       );                                                          \
3565       lval = (__typeof__(lval)) _res;                             \
3566    } while (0)
3567
3568 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3569                                   arg7,arg8,arg9,arg10,arg11)     \
3570    do {                                                           \
3571       volatile OrigFn        _orig = (orig);                      \
3572       volatile unsigned long _argvec[3+11];                       \
3573       volatile unsigned long _res;                                \
3574       /* _argvec[0] holds current r2 across the call */           \
3575       _argvec[1]   = (unsigned long)_orig.r2;                     \
3576       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3577       _argvec[2+1] = (unsigned long)arg1;                         \
3578       _argvec[2+2] = (unsigned long)arg2;                         \
3579       _argvec[2+3] = (unsigned long)arg3;                         \
3580       _argvec[2+4] = (unsigned long)arg4;                         \
3581       _argvec[2+5] = (unsigned long)arg5;                         \
3582       _argvec[2+6] = (unsigned long)arg6;                         \
3583       _argvec[2+7] = (unsigned long)arg7;                         \
3584       _argvec[2+8] = (unsigned long)arg8;                         \
3585       _argvec[2+9] = (unsigned long)arg9;                         \
3586       _argvec[2+10] = (unsigned long)arg10;                       \
3587       _argvec[2+11] = (unsigned long)arg11;                       \
3588       __asm__ volatile(                                           \
3589          "mr 11,%1\n\t"                                           \
3590          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3591          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3592          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3593          VG_EXPAND_FRAME_BY_trashes_r3(72)                        \
3594          /* arg11 */                                              \
3595          "lwz 3,44(11)\n\t"                                       \
3596          "stw 3,64(1)\n\t"                                        \
3597          /* arg10 */                                              \
3598          "lwz 3,40(11)\n\t"                                       \
3599          "stw 3,60(1)\n\t"                                        \
3600          /* arg9 */                                               \
3601          "lwz 3,36(11)\n\t"                                       \
3602          "stw 3,56(1)\n\t"                                        \
3603          /* args1-8 */                                            \
3604          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3605          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3606          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3607          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3608          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3609          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3610          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3611          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3612          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3613          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3614          "mr 11,%1\n\t"                                           \
3615          "mr %0,3\n\t"                                            \
3616          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3617          VG_CONTRACT_FRAME_BY(72)                                 \
3618          VG_CONTRACT_FRAME_BY(512)                                \
3619          : /*out*/   "=r" (_res)                                  \
3620          : /*in*/    "r" (&_argvec[2])                            \
3621          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3622       );                                                          \
3623       lval = (__typeof__(lval)) _res;                             \
3624    } while (0)
3625
3626 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3627                                 arg7,arg8,arg9,arg10,arg11,arg12) \
3628    do {                                                           \
3629       volatile OrigFn        _orig = (orig);                      \
3630       volatile unsigned long _argvec[3+12];                       \
3631       volatile unsigned long _res;                                \
3632       /* _argvec[0] holds current r2 across the call */           \
3633       _argvec[1]   = (unsigned long)_orig.r2;                     \
3634       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3635       _argvec[2+1] = (unsigned long)arg1;                         \
3636       _argvec[2+2] = (unsigned long)arg2;                         \
3637       _argvec[2+3] = (unsigned long)arg3;                         \
3638       _argvec[2+4] = (unsigned long)arg4;                         \
3639       _argvec[2+5] = (unsigned long)arg5;                         \
3640       _argvec[2+6] = (unsigned long)arg6;                         \
3641       _argvec[2+7] = (unsigned long)arg7;                         \
3642       _argvec[2+8] = (unsigned long)arg8;                         \
3643       _argvec[2+9] = (unsigned long)arg9;                         \
3644       _argvec[2+10] = (unsigned long)arg10;                       \
3645       _argvec[2+11] = (unsigned long)arg11;                       \
3646       _argvec[2+12] = (unsigned long)arg12;                       \
3647       __asm__ volatile(                                           \
3648          "mr 11,%1\n\t"                                           \
3649          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3650          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3651          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3652          VG_EXPAND_FRAME_BY_trashes_r3(72)                        \
3653          /* arg12 */                                              \
3654          "lwz 3,48(11)\n\t"                                       \
3655          "stw 3,68(1)\n\t"                                        \
3656          /* arg11 */                                              \
3657          "lwz 3,44(11)\n\t"                                       \
3658          "stw 3,64(1)\n\t"                                        \
3659          /* arg10 */                                              \
3660          "lwz 3,40(11)\n\t"                                       \
3661          "stw 3,60(1)\n\t"                                        \
3662          /* arg9 */                                               \
3663          "lwz 3,36(11)\n\t"                                       \
3664          "stw 3,56(1)\n\t"                                        \
3665          /* args1-8 */                                            \
3666          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3667          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3668          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3669          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3670          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3671          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3672          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3673          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3674          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3675          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3676          "mr 11,%1\n\t"                                           \
3677          "mr %0,3\n\t"                                            \
3678          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3679          VG_CONTRACT_FRAME_BY(72)                                 \
3680          VG_CONTRACT_FRAME_BY(512)                                \
3681          : /*out*/   "=r" (_res)                                  \
3682          : /*in*/    "r" (&_argvec[2])                            \
3683          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3684       );                                                          \
3685       lval = (__typeof__(lval)) _res;                             \
3686    } while (0)
3687
3688 #endif /* PLAT_ppc32_aix5 */
3689
3690 /* ------------------------ ppc64-aix5 ------------------------- */
3691
3692 #if defined(PLAT_ppc64_aix5)
3693
3694 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
3695
3696 /* These regs are trashed by the hidden call. */
3697 #define __CALLER_SAVED_REGS                                       \
3698    "lr", "ctr", "xer",                                            \
3699    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
3700    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
3701    "r11", "r12", "r13"
3702
3703 /* Expand the stack frame, copying enough info that unwinding
3704    still works.  Trashes r3. */
3705
3706 #define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr)                      \
3707          "addi 1,1,-" #_n_fr "\n\t"                               \
3708          "ld   3," #_n_fr "(1)\n\t"                               \
3709          "std  3,0(1)\n\t"
3710
3711 #define VG_CONTRACT_FRAME_BY(_n_fr)                               \
3712          "addi 1,1," #_n_fr "\n\t"
3713
3714 /* These CALL_FN_ macros assume that on ppc64-aix5, sizeof(unsigned
3715    long) == 8. */
3716
3717 #define CALL_FN_W_v(lval, orig)                                   \
3718    do {                                                           \
3719       volatile OrigFn        _orig = (orig);                      \
3720       volatile unsigned long _argvec[3+0];                        \
3721       volatile unsigned long _res;                                \
3722       /* _argvec[0] holds current r2 across the call */           \
3723       _argvec[1] = (unsigned long)_orig.r2;                       \
3724       _argvec[2] = (unsigned long)_orig.nraddr;                   \
3725       __asm__ volatile(                                           \
3726          "mr 11,%1\n\t"                                           \
3727          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3728          "std  2,-16(11)\n\t" /* save tocptr */                   \
3729          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3730          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3731          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3732          "mr 11,%1\n\t"                                           \
3733          "mr %0,3\n\t"                                            \
3734          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3735          VG_CONTRACT_FRAME_BY(512)                                \
3736          : /*out*/   "=r" (_res)                                  \
3737          : /*in*/    "r" (&_argvec[2])                            \
3738          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3739       );                                                          \
3740       lval = (__typeof__(lval)) _res;                             \
3741    } while (0)
3742
3743 #define CALL_FN_W_W(lval, orig, arg1)                             \
3744    do {                                                           \
3745       volatile OrigFn        _orig = (orig);                      \
3746       volatile unsigned long _argvec[3+1];                        \
3747       volatile unsigned long _res;                                \
3748       /* _argvec[0] holds current r2 across the call */           \
3749       _argvec[1]   = (unsigned long)_orig.r2;                     \
3750       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3751       _argvec[2+1] = (unsigned long)arg1;                         \
3752       __asm__ volatile(                                           \
3753          "mr 11,%1\n\t"                                           \
3754          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3755          "std  2,-16(11)\n\t" /* save tocptr */                   \
3756          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3757          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3758          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3759          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3760          "mr 11,%1\n\t"                                           \
3761          "mr %0,3\n\t"                                            \
3762          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3763          VG_CONTRACT_FRAME_BY(512)                                \
3764          : /*out*/   "=r" (_res)                                  \
3765          : /*in*/    "r" (&_argvec[2])                            \
3766          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3767       );                                                          \
3768       lval = (__typeof__(lval)) _res;                             \
3769    } while (0)
3770
3771 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3772    do {                                                           \
3773       volatile OrigFn        _orig = (orig);                      \
3774       volatile unsigned long _argvec[3+2];                        \
3775       volatile unsigned long _res;                                \
3776       /* _argvec[0] holds current r2 across the call */           \
3777       _argvec[1]   = (unsigned long)_orig.r2;                     \
3778       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3779       _argvec[2+1] = (unsigned long)arg1;                         \
3780       _argvec[2+2] = (unsigned long)arg2;                         \
3781       __asm__ volatile(                                           \
3782          "mr 11,%1\n\t"                                           \
3783          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3784          "std  2,-16(11)\n\t" /* save tocptr */                   \
3785          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3786          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3787          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3788          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3789          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3790          "mr 11,%1\n\t"                                           \
3791          "mr %0,3\n\t"                                            \
3792          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3793          VG_CONTRACT_FRAME_BY(512)                                \
3794          : /*out*/   "=r" (_res)                                  \
3795          : /*in*/    "r" (&_argvec[2])                            \
3796          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3797       );                                                          \
3798       lval = (__typeof__(lval)) _res;                             \
3799    } while (0)
3800
3801 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3802    do {                                                           \
3803       volatile OrigFn        _orig = (orig);                      \
3804       volatile unsigned long _argvec[3+3];                        \
3805       volatile unsigned long _res;                                \
3806       /* _argvec[0] holds current r2 across the call */           \
3807       _argvec[1]   = (unsigned long)_orig.r2;                     \
3808       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3809       _argvec[2+1] = (unsigned long)arg1;                         \
3810       _argvec[2+2] = (unsigned long)arg2;                         \
3811       _argvec[2+3] = (unsigned long)arg3;                         \
3812       __asm__ volatile(                                           \
3813          "mr 11,%1\n\t"                                           \
3814          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3815          "std  2,-16(11)\n\t" /* save tocptr */                   \
3816          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3817          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3818          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3819          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3820          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3821          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3822          "mr 11,%1\n\t"                                           \
3823          "mr %0,3\n\t"                                            \
3824          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3825          VG_CONTRACT_FRAME_BY(512)                                \
3826          : /*out*/   "=r" (_res)                                  \
3827          : /*in*/    "r" (&_argvec[2])                            \
3828          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3829       );                                                          \
3830       lval = (__typeof__(lval)) _res;                             \
3831    } while (0)
3832
3833 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3834    do {                                                           \
3835       volatile OrigFn        _orig = (orig);                      \
3836       volatile unsigned long _argvec[3+4];                        \
3837       volatile unsigned long _res;                                \
3838       /* _argvec[0] holds current r2 across the call */           \
3839       _argvec[1]   = (unsigned long)_orig.r2;                     \
3840       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3841       _argvec[2+1] = (unsigned long)arg1;                         \
3842       _argvec[2+2] = (unsigned long)arg2;                         \
3843       _argvec[2+3] = (unsigned long)arg3;                         \
3844       _argvec[2+4] = (unsigned long)arg4;                         \
3845       __asm__ volatile(                                           \
3846          "mr 11,%1\n\t"                                           \
3847          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3848          "std  2,-16(11)\n\t" /* save tocptr */                   \
3849          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3850          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3851          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3852          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3853          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3854          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3855          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3856          "mr 11,%1\n\t"                                           \
3857          "mr %0,3\n\t"                                            \
3858          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3859          VG_CONTRACT_FRAME_BY(512)                                \
3860          : /*out*/   "=r" (_res)                                  \
3861          : /*in*/    "r" (&_argvec[2])                            \
3862          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3863       );                                                          \
3864       lval = (__typeof__(lval)) _res;                             \
3865    } while (0)
3866
3867 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3868    do {                                                           \
3869       volatile OrigFn        _orig = (orig);                      \
3870       volatile unsigned long _argvec[3+5];                        \
3871       volatile unsigned long _res;                                \
3872       /* _argvec[0] holds current r2 across the call */           \
3873       _argvec[1]   = (unsigned long)_orig.r2;                     \
3874       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3875       _argvec[2+1] = (unsigned long)arg1;                         \
3876       _argvec[2+2] = (unsigned long)arg2;                         \
3877       _argvec[2+3] = (unsigned long)arg3;                         \
3878       _argvec[2+4] = (unsigned long)arg4;                         \
3879       _argvec[2+5] = (unsigned long)arg5;                         \
3880       __asm__ volatile(                                           \
3881          "mr 11,%1\n\t"                                           \
3882          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3883          "std  2,-16(11)\n\t" /* save tocptr */                   \
3884          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3885          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3886          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3887          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3888          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3889          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3890          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3891          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3892          "mr 11,%1\n\t"                                           \
3893          "mr %0,3\n\t"                                            \
3894          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3895          VG_CONTRACT_FRAME_BY(512)                                \
3896          : /*out*/   "=r" (_res)                                  \
3897          : /*in*/    "r" (&_argvec[2])                            \
3898          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3899       );                                                          \
3900       lval = (__typeof__(lval)) _res;                             \
3901    } while (0)
3902
3903 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3904    do {                                                           \
3905       volatile OrigFn        _orig = (orig);                      \
3906       volatile unsigned long _argvec[3+6];                        \
3907       volatile unsigned long _res;                                \
3908       /* _argvec[0] holds current r2 across the call */           \
3909       _argvec[1]   = (unsigned long)_orig.r2;                     \
3910       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3911       _argvec[2+1] = (unsigned long)arg1;                         \
3912       _argvec[2+2] = (unsigned long)arg2;                         \
3913       _argvec[2+3] = (unsigned long)arg3;                         \
3914       _argvec[2+4] = (unsigned long)arg4;                         \
3915       _argvec[2+5] = (unsigned long)arg5;                         \
3916       _argvec[2+6] = (unsigned long)arg6;                         \
3917       __asm__ volatile(                                           \
3918          "mr 11,%1\n\t"                                           \
3919          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3920          "std  2,-16(11)\n\t" /* save tocptr */                   \
3921          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3922          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3923          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3924          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3925          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3926          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3927          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3928          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3929          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3930          "mr 11,%1\n\t"                                           \
3931          "mr %0,3\n\t"                                            \
3932          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3933          VG_CONTRACT_FRAME_BY(512)                                \
3934          : /*out*/   "=r" (_res)                                  \
3935          : /*in*/    "r" (&_argvec[2])                            \
3936          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3937       );                                                          \
3938       lval = (__typeof__(lval)) _res;                             \
3939    } while (0)
3940
3941 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3942                                  arg7)                            \
3943    do {                                                           \
3944       volatile OrigFn        _orig = (orig);                      \
3945       volatile unsigned long _argvec[3+7];                        \
3946       volatile unsigned long _res;                                \
3947       /* _argvec[0] holds current r2 across the call */           \
3948       _argvec[1]   = (unsigned long)_orig.r2;                     \
3949       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3950       _argvec[2+1] = (unsigned long)arg1;                         \
3951       _argvec[2+2] = (unsigned long)arg2;                         \
3952       _argvec[2+3] = (unsigned long)arg3;                         \
3953       _argvec[2+4] = (unsigned long)arg4;                         \
3954       _argvec[2+5] = (unsigned long)arg5;                         \
3955       _argvec[2+6] = (unsigned long)arg6;                         \
3956       _argvec[2+7] = (unsigned long)arg7;                         \
3957       __asm__ volatile(                                           \
3958          "mr 11,%1\n\t"                                           \
3959          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3960          "std  2,-16(11)\n\t" /* save tocptr */                   \
3961          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3962          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3963          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3964          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3965          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3966          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3967          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3968          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3969          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3970          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3971          "mr 11,%1\n\t"                                           \
3972          "mr %0,3\n\t"                                            \
3973          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3974          VG_CONTRACT_FRAME_BY(512)                                \
3975          : /*out*/   "=r" (_res)                                  \
3976          : /*in*/    "r" (&_argvec[2])                            \
3977          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3978       );                                                          \
3979       lval = (__typeof__(lval)) _res;                             \
3980    } while (0)
3981
3982 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3983                                  arg7,arg8)                       \
3984    do {                                                           \
3985       volatile OrigFn        _orig = (orig);                      \
3986       volatile unsigned long _argvec[3+8];                        \
3987       volatile unsigned long _res;                                \
3988       /* _argvec[0] holds current r2 across the call */           \
3989       _argvec[1]   = (unsigned long)_orig.r2;                     \
3990       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3991       _argvec[2+1] = (unsigned long)arg1;                         \
3992       _argvec[2+2] = (unsigned long)arg2;                         \
3993       _argvec[2+3] = (unsigned long)arg3;                         \
3994       _argvec[2+4] = (unsigned long)arg4;                         \
3995       _argvec[2+5] = (unsigned long)arg5;                         \
3996       _argvec[2+6] = (unsigned long)arg6;                         \
3997       _argvec[2+7] = (unsigned long)arg7;                         \
3998       _argvec[2+8] = (unsigned long)arg8;                         \
3999       __asm__ volatile(                                           \
4000          "mr 11,%1\n\t"                                           \
4001          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4002          "std  2,-16(11)\n\t" /* save tocptr */                   \
4003          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4004          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4005          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4006          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4007          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4008          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4009          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4010          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4011          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4012          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4013          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4014          "mr 11,%1\n\t"                                           \
4015          "mr %0,3\n\t"                                            \
4016          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4017          VG_CONTRACT_FRAME_BY(512)                                \
4018          : /*out*/   "=r" (_res)                                  \
4019          : /*in*/    "r" (&_argvec[2])                            \
4020          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4021       );                                                          \
4022       lval = (__typeof__(lval)) _res;                             \
4023    } while (0)
4024
4025 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4026                                  arg7,arg8,arg9)                  \
4027    do {                                                           \
4028       volatile OrigFn        _orig = (orig);                      \
4029       volatile unsigned long _argvec[3+9];                        \
4030       volatile unsigned long _res;                                \
4031       /* _argvec[0] holds current r2 across the call */           \
4032       _argvec[1]   = (unsigned long)_orig.r2;                     \
4033       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4034       _argvec[2+1] = (unsigned long)arg1;                         \
4035       _argvec[2+2] = (unsigned long)arg2;                         \
4036       _argvec[2+3] = (unsigned long)arg3;                         \
4037       _argvec[2+4] = (unsigned long)arg4;                         \
4038       _argvec[2+5] = (unsigned long)arg5;                         \
4039       _argvec[2+6] = (unsigned long)arg6;                         \
4040       _argvec[2+7] = (unsigned long)arg7;                         \
4041       _argvec[2+8] = (unsigned long)arg8;                         \
4042       _argvec[2+9] = (unsigned long)arg9;                         \
4043       __asm__ volatile(                                           \
4044          "mr 11,%1\n\t"                                           \
4045          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4046          "std  2,-16(11)\n\t" /* save tocptr */                   \
4047          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4048          VG_EXPAND_FRAME_BY_trashes_r3(128)                       \
4049          /* arg9 */                                               \
4050          "ld  3,72(11)\n\t"                                       \
4051          "std 3,112(1)\n\t"                                       \
4052          /* args1-8 */                                            \
4053          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4054          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4055          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4056          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4057          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4058          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4059          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4060          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4061          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4062          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4063          "mr 11,%1\n\t"                                           \
4064          "mr %0,3\n\t"                                            \
4065          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4066          VG_CONTRACT_FRAME_BY(128)                                \
4067          VG_CONTRACT_FRAME_BY(512)                                \
4068          : /*out*/   "=r" (_res)                                  \
4069          : /*in*/    "r" (&_argvec[2])                            \
4070          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4071       );                                                          \
4072       lval = (__typeof__(lval)) _res;                             \
4073    } while (0)
4074
4075 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4076                                   arg7,arg8,arg9,arg10)           \
4077    do {                                                           \
4078       volatile OrigFn        _orig = (orig);                      \
4079       volatile unsigned long _argvec[3+10];                       \
4080       volatile unsigned long _res;                                \
4081       /* _argvec[0] holds current r2 across the call */           \
4082       _argvec[1]   = (unsigned long)_orig.r2;                     \
4083       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4084       _argvec[2+1] = (unsigned long)arg1;                         \
4085       _argvec[2+2] = (unsigned long)arg2;                         \
4086       _argvec[2+3] = (unsigned long)arg3;                         \
4087       _argvec[2+4] = (unsigned long)arg4;                         \
4088       _argvec[2+5] = (unsigned long)arg5;                         \
4089       _argvec[2+6] = (unsigned long)arg6;                         \
4090       _argvec[2+7] = (unsigned long)arg7;                         \
4091       _argvec[2+8] = (unsigned long)arg8;                         \
4092       _argvec[2+9] = (unsigned long)arg9;                         \
4093       _argvec[2+10] = (unsigned long)arg10;                       \
4094       __asm__ volatile(                                           \
4095          "mr 11,%1\n\t"                                           \
4096          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4097          "std  2,-16(11)\n\t" /* save tocptr */                   \
4098          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4099          VG_EXPAND_FRAME_BY_trashes_r3(128)                       \
4100          /* arg10 */                                              \
4101          "ld  3,80(11)\n\t"                                       \
4102          "std 3,120(1)\n\t"                                       \
4103          /* arg9 */                                               \
4104          "ld  3,72(11)\n\t"                                       \
4105          "std 3,112(1)\n\t"                                       \
4106          /* args1-8 */                                            \
4107          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4108          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4109          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4110          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4111          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4112          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4113          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4114          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4115          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4116          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4117          "mr 11,%1\n\t"                                           \
4118          "mr %0,3\n\t"                                            \
4119          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4120          VG_CONTRACT_FRAME_BY(128)                                \
4121          VG_CONTRACT_FRAME_BY(512)                                \
4122          : /*out*/   "=r" (_res)                                  \
4123          : /*in*/    "r" (&_argvec[2])                            \
4124          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4125       );                                                          \
4126       lval = (__typeof__(lval)) _res;                             \
4127    } while (0)
4128
4129 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4130                                   arg7,arg8,arg9,arg10,arg11)     \
4131    do {                                                           \
4132       volatile OrigFn        _orig = (orig);                      \
4133       volatile unsigned long _argvec[3+11];                       \
4134       volatile unsigned long _res;                                \
4135       /* _argvec[0] holds current r2 across the call */           \
4136       _argvec[1]   = (unsigned long)_orig.r2;                     \
4137       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4138       _argvec[2+1] = (unsigned long)arg1;                         \
4139       _argvec[2+2] = (unsigned long)arg2;                         \
4140       _argvec[2+3] = (unsigned long)arg3;                         \
4141       _argvec[2+4] = (unsigned long)arg4;                         \
4142       _argvec[2+5] = (unsigned long)arg5;                         \
4143       _argvec[2+6] = (unsigned long)arg6;                         \
4144       _argvec[2+7] = (unsigned long)arg7;                         \
4145       _argvec[2+8] = (unsigned long)arg8;                         \
4146       _argvec[2+9] = (unsigned long)arg9;                         \
4147       _argvec[2+10] = (unsigned long)arg10;                       \
4148       _argvec[2+11] = (unsigned long)arg11;                       \
4149       __asm__ volatile(                                           \
4150          "mr 11,%1\n\t"                                           \
4151          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4152          "std  2,-16(11)\n\t" /* save tocptr */                   \
4153          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4154          VG_EXPAND_FRAME_BY_trashes_r3(144)                       \
4155          /* arg11 */                                              \
4156          "ld  3,88(11)\n\t"                                       \
4157          "std 3,128(1)\n\t"                                       \
4158          /* arg10 */                                              \
4159          "ld  3,80(11)\n\t"                                       \
4160          "std 3,120(1)\n\t"                                       \
4161          /* arg9 */                                               \
4162          "ld  3,72(11)\n\t"                                       \
4163          "std 3,112(1)\n\t"                                       \
4164          /* args1-8 */                                            \
4165          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4166          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4167          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4168          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4169          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4170          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4171          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4172          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4173          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4174          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4175          "mr 11,%1\n\t"                                           \
4176          "mr %0,3\n\t"                                            \
4177          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4178          VG_CONTRACT_FRAME_BY(144)                                \
4179          VG_CONTRACT_FRAME_BY(512)                                \
4180          : /*out*/   "=r" (_res)                                  \
4181          : /*in*/    "r" (&_argvec[2])                            \
4182          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4183       );                                                          \
4184       lval = (__typeof__(lval)) _res;                             \
4185    } while (0)
4186
4187 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4188                                 arg7,arg8,arg9,arg10,arg11,arg12) \
4189    do {                                                           \
4190       volatile OrigFn        _orig = (orig);                      \
4191       volatile unsigned long _argvec[3+12];                       \
4192       volatile unsigned long _res;                                \
4193       /* _argvec[0] holds current r2 across the call */           \
4194       _argvec[1]   = (unsigned long)_orig.r2;                     \
4195       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4196       _argvec[2+1] = (unsigned long)arg1;                         \
4197       _argvec[2+2] = (unsigned long)arg2;                         \
4198       _argvec[2+3] = (unsigned long)arg3;                         \
4199       _argvec[2+4] = (unsigned long)arg4;                         \
4200       _argvec[2+5] = (unsigned long)arg5;                         \
4201       _argvec[2+6] = (unsigned long)arg6;                         \
4202       _argvec[2+7] = (unsigned long)arg7;                         \
4203       _argvec[2+8] = (unsigned long)arg8;                         \
4204       _argvec[2+9] = (unsigned long)arg9;                         \
4205       _argvec[2+10] = (unsigned long)arg10;                       \
4206       _argvec[2+11] = (unsigned long)arg11;                       \
4207       _argvec[2+12] = (unsigned long)arg12;                       \
4208       __asm__ volatile(                                           \
4209          "mr 11,%1\n\t"                                           \
4210          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4211          "std  2,-16(11)\n\t" /* save tocptr */                   \
4212          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4213          VG_EXPAND_FRAME_BY_trashes_r3(144)                       \
4214          /* arg12 */                                              \
4215          "ld  3,96(11)\n\t"                                       \
4216          "std 3,136(1)\n\t"                                       \
4217          /* arg11 */                                              \
4218          "ld  3,88(11)\n\t"                                       \
4219          "std 3,128(1)\n\t"                                       \
4220          /* arg10 */                                              \
4221          "ld  3,80(11)\n\t"                                       \
4222          "std 3,120(1)\n\t"                                       \
4223          /* arg9 */                                               \
4224          "ld  3,72(11)\n\t"                                       \
4225          "std 3,112(1)\n\t"                                       \
4226          /* args1-8 */                                            \
4227          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4228          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4229          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4230          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4231          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4232          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4233          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4234          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4235          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4236          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4237          "mr 11,%1\n\t"                                           \
4238          "mr %0,3\n\t"                                            \
4239          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4240          VG_CONTRACT_FRAME_BY(144)                                \
4241          VG_CONTRACT_FRAME_BY(512)                                \
4242          : /*out*/   "=r" (_res)                                  \
4243          : /*in*/    "r" (&_argvec[2])                            \
4244          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4245       );                                                          \
4246       lval = (__typeof__(lval)) _res;                             \
4247    } while (0)
4248
4249 #endif /* PLAT_ppc64_aix5 */
4250
4251
4252 /* ------------------------------------------------------------------ */
4253 /* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS.               */
4254 /*                                                                    */
4255 /* ------------------------------------------------------------------ */
4256
4257 /* Some request codes.  There are many more of these, but most are not
4258    exposed to end-user view.  These are the public ones, all of the
4259    form 0x1000 + small_number.
4260
4261    Core ones are in the range 0x00000000--0x0000ffff.  The non-public
4262    ones start at 0x2000.
4263 */
4264
4265 /* These macros are used by tools -- they must be public, but don't
4266    embed them into other programs. */
4267 #define VG_USERREQ_TOOL_BASE(a,b) \
4268    ((unsigned int)(((a)&0xff) << 24 | ((b)&0xff) << 16))
4269 #define VG_IS_TOOL_USERREQ(a, b, v) \
4270    (VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000))
4271
4272 /* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !!
4273    This enum comprises an ABI exported by Valgrind to programs
4274    which use client requests.  DO NOT CHANGE THE ORDER OF THESE
4275    ENTRIES, NOR DELETE ANY -- add new ones at the end. */
4276 typedef
4277    enum { VG_USERREQ__RUNNING_ON_VALGRIND  = 0x1001,
4278           VG_USERREQ__DISCARD_TRANSLATIONS = 0x1002,
4279
4280           /* These allow any function to be called from the simulated
4281              CPU but run on the real CPU.  Nb: the first arg passed to
4282              the function is always the ThreadId of the running
4283              thread!  So CLIENT_CALL0 actually requires a 1 arg
4284              function, etc. */
4285           VG_USERREQ__CLIENT_CALL0 = 0x1101,
4286           VG_USERREQ__CLIENT_CALL1 = 0x1102,
4287           VG_USERREQ__CLIENT_CALL2 = 0x1103,
4288           VG_USERREQ__CLIENT_CALL3 = 0x1104,
4289
4290           /* Can be useful in regression testing suites -- eg. can
4291              send Valgrind's output to /dev/null and still count
4292              errors. */
4293           VG_USERREQ__COUNT_ERRORS = 0x1201,
4294
4295           /* These are useful and can be interpreted by any tool that
4296              tracks malloc() et al, by using vg_replace_malloc.c. */
4297           VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301,
4298           VG_USERREQ__FREELIKE_BLOCK   = 0x1302,
4299           /* Memory pool support. */
4300           VG_USERREQ__CREATE_MEMPOOL   = 0x1303,
4301           VG_USERREQ__DESTROY_MEMPOOL  = 0x1304,
4302           VG_USERREQ__MEMPOOL_ALLOC    = 0x1305,
4303           VG_USERREQ__MEMPOOL_FREE     = 0x1306,
4304           VG_USERREQ__MEMPOOL_TRIM     = 0x1307,
4305           VG_USERREQ__MOVE_MEMPOOL     = 0x1308,
4306           VG_USERREQ__MEMPOOL_CHANGE   = 0x1309,
4307           VG_USERREQ__MEMPOOL_EXISTS   = 0x130a,
4308
4309           /* Allow printfs to valgrind log. */
4310           /* The first two pass the va_list argument by value, which
4311              assumes it is the same size as or smaller than a UWord,
4312              which generally isn't the case.  Hence are deprecated.
4313              The second two pass the vargs by reference and so are
4314              immune to this problem. */
4315           /* both :: char* fmt, va_list vargs (DEPRECATED) */
4316           VG_USERREQ__PRINTF           = 0x1401,
4317           VG_USERREQ__PRINTF_BACKTRACE = 0x1402,
4318           /* both :: char* fmt, va_list* vargs */
4319           VG_USERREQ__PRINTF_VALIST_BY_REF = 0x1403,
4320           VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF = 0x1404,
4321
4322           /* Stack support. */
4323           VG_USERREQ__STACK_REGISTER   = 0x1501,
4324           VG_USERREQ__STACK_DEREGISTER = 0x1502,
4325           VG_USERREQ__STACK_CHANGE     = 0x1503,
4326
4327           /* Wine support */
4328           VG_USERREQ__LOAD_PDB_DEBUGINFO = 0x1601,
4329
4330           /* Querying of debug info. */
4331           VG_USERREQ__MAP_IP_TO_SRCLOC = 0x1701
4332    } Vg_ClientRequest;
4333
4334 #if !defined(__GNUC__)
4335 #  define __extension__ /* */
4336 #endif
4337
4338
4339 /*
4340  * VALGRIND_DO_CLIENT_REQUEST_EXPR(): a C expression that invokes a Valgrind
4341  * client request and whose value equals the client request result.
4342  */
4343
4344 #if defined(NVALGRIND)
4345
4346 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                               \
4347         _zzq_default, _zzq_request,                                    \
4348         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)         \
4349    (_zzq_default)
4350
4351 #else /*defined(NVALGRIND)*/
4352
4353 #if defined(_MSC_VER)
4354
4355 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                                \
4356         _zzq_default, _zzq_request,                                     \
4357         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)          \
4358    (vg_VALGRIND_DO_CLIENT_REQUEST_EXPR((uintptr_t)(_zzq_default),       \
4359         (_zzq_request), (uintptr_t)(_zzq_arg1), (uintptr_t)(_zzq_arg2), \
4360         (uintptr_t)(_zzq_arg3), (uintptr_t)(_zzq_arg4),                 \
4361         (uintptr_t)(_zzq_arg5)))
4362
4363 static __inline unsigned
4364 vg_VALGRIND_DO_CLIENT_REQUEST_EXPR(uintptr_t _zzq_default,
4365                                    unsigned _zzq_request, uintptr_t _zzq_arg1,
4366                                    uintptr_t _zzq_arg2, uintptr_t _zzq_arg3,
4367                                    uintptr_t _zzq_arg4, uintptr_t _zzq_arg5)
4368 {
4369     unsigned _zzq_rlval;
4370     VALGRIND_DO_CLIENT_REQUEST(_zzq_rlval, _zzq_default, _zzq_request,
4371                       _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5);
4372     return _zzq_rlval;
4373 }
4374
4375 #else /*defined(_MSC_VER)*/
4376
4377 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                               \
4378         _zzq_default, _zzq_request,                                    \
4379         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)         \
4380    (__extension__({unsigned int _zzq_rlval;                            \
4381     VALGRIND_DO_CLIENT_REQUEST(_zzq_rlval, _zzq_default, _zzq_request, \
4382                 _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
4383     _zzq_rlval;                                                        \
4384    }))
4385
4386 #endif /*defined(_MSC_VER)*/
4387
4388 #endif /*defined(NVALGRIND)*/
4389
4390
4391 /* Returns the number of Valgrinds this code is running under.  That
4392    is, 0 if running natively, 1 if running under Valgrind, 2 if
4393    running under Valgrind which is running under another Valgrind,
4394    etc. */
4395 #define RUNNING_ON_VALGRIND                                           \
4396     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* if not */,                   \
4397                                     VG_USERREQ__RUNNING_ON_VALGRIND,  \
4398                                     0, 0, 0, 0, 0)                    \
4399
4400
4401 /* Discard translation of code in the range [_qzz_addr .. _qzz_addr +
4402    _qzz_len - 1].  Useful if you are debugging a JITter or some such,
4403    since it provides a way to make sure valgrind will retranslate the
4404    invalidated area.  Returns no value. */
4405 #define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len)         \
4406    {unsigned int _qzz_res;                                        \
4407     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4408                                VG_USERREQ__DISCARD_TRANSLATIONS,  \
4409                                _qzz_addr, _qzz_len, 0, 0, 0);     \
4410    }
4411
4412
4413 /* These requests are for getting Valgrind itself to print something.
4414    Possibly with a backtrace.  This is a really ugly hack.  The return value
4415    is the number of characters printed, excluding the "**<pid>** " part at the
4416    start and the backtrace (if present). */
4417
4418 #if defined(NVALGRIND)
4419
4420 /* In Exim the following two lines have been changed from the original
4421    version for portability to C89 compilers that don't support variable
4422    argument macros. We don't use these macros so it doesn't matter much what
4423    we do with them, but the following will work OK in most situations though
4424    it may cause complaints about expressions without side-effects. */
4425 #  define VALGRIND_PRINTF (void)
4426 #  define VALGRIND_PRINTF_BACKTRACE (void)
4427
4428 #else /* NVALGRIND */
4429
4430 #if !defined(_MSC_VER)
4431 /* Modern GCC will optimize the static routine out if unused,
4432    and unused attribute will shut down warnings about it.  */
4433 static int VALGRIND_PRINTF(const char *format, ...)
4434    __attribute__((format(__printf__, 1, 2), __unused__));
4435 #endif
4436 static int
4437 #if defined(_MSC_VER)
4438 __inline
4439 #endif
4440 VALGRIND_PRINTF(const char *format, ...)
4441 {
4442    unsigned long _qzz_res;
4443    va_list vargs;
4444    va_start(vargs, format);
4445 #if defined(_MSC_VER)
4446    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,
4447                               VG_USERREQ__PRINTF_VALIST_BY_REF,
4448                               (uintptr_t)format,
4449                               (uintptr_t)&vargs,
4450                               0, 0, 0);
4451 #else
4452    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,
4453                               VG_USERREQ__PRINTF_VALIST_BY_REF,
4454                               (unsigned long)format,
4455                               (unsigned long)&vargs,
4456                               0, 0, 0);
4457 #endif
4458    va_end(vargs);
4459    return (int)_qzz_res;
4460 }
4461
4462 #if !defined(_MSC_VER)
4463 static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
4464    __attribute__((format(__printf__, 1, 2), __unused__));
4465 #endif
4466 static int
4467 #if defined(_MSC_VER)
4468 __inline
4469 #endif
4470 VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
4471 {
4472    unsigned long _qzz_res;
4473    va_list vargs;
4474    va_start(vargs, format);
4475 #if defined(_MSC_VER)
4476    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,
4477                               VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
4478                               (uintptr_t)format,
4479                               (uintptr_t)&vargs,
4480                               0, 0, 0);
4481 #else
4482    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,
4483                               VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
4484                               (unsigned long)format,
4485                               (unsigned long)&vargs,
4486                               0, 0, 0);
4487 #endif
4488    va_end(vargs);
4489    return (int)_qzz_res;
4490 }
4491
4492 #endif /* NVALGRIND */
4493
4494
4495 /* These requests allow control to move from the simulated CPU to the
4496    real CPU, calling an arbitary function.
4497
4498    Note that the current ThreadId is inserted as the first argument.
4499    So this call:
4500
4501      VALGRIND_NON_SIMD_CALL2(f, arg1, arg2)
4502
4503    requires f to have this signature:
4504
4505      Word f(Word tid, Word arg1, Word arg2)
4506
4507    where "Word" is a word-sized type.
4508
4509    Note that these client requests are not entirely reliable.  For example,
4510    if you call a function with them that subsequently calls printf(),
4511    there's a high chance Valgrind will crash.  Generally, your prospects of
4512    these working are made higher if the called function does not refer to
4513    any global variables, and does not refer to any libc or other functions
4514    (printf et al).  Any kind of entanglement with libc or dynamic linking is
4515    likely to have a bad outcome, for tricky reasons which we've grappled
4516    with a lot in the past.
4517 */
4518 #define VALGRIND_NON_SIMD_CALL0(_qyy_fn)                          \
4519    __extension__                                                  \
4520    ({unsigned long _qyy_res;                                      \
4521     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
4522                                VG_USERREQ__CLIENT_CALL0,          \
4523                                _qyy_fn,                           \
4524                                0, 0, 0, 0);                       \
4525     _qyy_res;                                                     \
4526    })
4527
4528 #define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1)               \
4529    __extension__                                                  \
4530    ({unsigned long _qyy_res;                                      \
4531     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
4532                                VG_USERREQ__CLIENT_CALL1,          \
4533                                _qyy_fn,                           \
4534                                _qyy_arg1, 0, 0, 0);               \
4535     _qyy_res;                                                     \
4536    })
4537
4538 #define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2)    \
4539    __extension__                                                  \
4540    ({unsigned long _qyy_res;                                      \
4541     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
4542                                VG_USERREQ__CLIENT_CALL2,          \
4543                                _qyy_fn,                           \
4544                                _qyy_arg1, _qyy_arg2, 0, 0);       \
4545     _qyy_res;                                                     \
4546    })
4547
4548 #define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \
4549    __extension__                                                  \
4550    ({unsigned long _qyy_res;                                      \
4551     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
4552                                VG_USERREQ__CLIENT_CALL3,          \
4553                                _qyy_fn,                           \
4554                                _qyy_arg1, _qyy_arg2,              \
4555                                _qyy_arg3, 0);                     \
4556     _qyy_res;                                                     \
4557    })
4558
4559
4560 /* Counts the number of errors that have been recorded by a tool.  Nb:
4561    the tool must record the errors with VG_(maybe_record_error)() or
4562    VG_(unique_error)() for them to be counted. */
4563 #define VALGRIND_COUNT_ERRORS                                     \
4564    __extension__                                                  \
4565    ({unsigned int _qyy_res;                                       \
4566     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
4567                                VG_USERREQ__COUNT_ERRORS,          \
4568                                0, 0, 0, 0, 0);                    \
4569     _qyy_res;                                                     \
4570    })
4571
4572 /* Several Valgrind tools (Memcheck, Massif, Helgrind, DRD) rely on knowing
4573    when heap blocks are allocated in order to give accurate results.  This
4574    happens automatically for the standard allocator functions such as
4575    malloc(), calloc(), realloc(), memalign(), new, new[], free(), delete,
4576    delete[], etc.
4577
4578    But if your program uses a custom allocator, this doesn't automatically
4579    happen, and Valgrind will not do as well.  For example, if you allocate
4580    superblocks with mmap() and then allocates chunks of the superblocks, all
4581    Valgrind's observations will be at the mmap() level and it won't know that
4582    the chunks should be considered separate entities.  In Memcheck's case,
4583    that means you probably won't get heap block overrun detection (because
4584    there won't be redzones marked as unaddressable) and you definitely won't
4585    get any leak detection.
4586
4587    The following client requests allow a custom allocator to be annotated so
4588    that it can be handled accurately by Valgrind.
4589
4590    VALGRIND_MALLOCLIKE_BLOCK marks a region of memory as having been allocated
4591    by a malloc()-like function.  For Memcheck (an illustrative case), this
4592    does two things:
4593
4594    - It records that the block has been allocated.  This means any addresses
4595      within the block mentioned in error messages will be
4596      identified as belonging to the block.  It also means that if the block
4597      isn't freed it will be detected by the leak checker.
4598
4599    - It marks the block as being addressable and undefined (if 'is_zeroed' is
4600      not set), or addressable and defined (if 'is_zeroed' is set).  This
4601      controls how accesses to the block by the program are handled.
4602
4603    'addr' is the start of the usable block (ie. after any
4604    redzone), 'sizeB' is its size.  'rzB' is the redzone size if the allocator
4605    can apply redzones -- these are blocks of padding at the start and end of
4606    each block.  Adding redzones is recommended as it makes it much more likely
4607    Valgrind will spot block overruns.  `is_zeroed' indicates if the memory is
4608    zeroed (or filled with another predictable value), as is the case for
4609    calloc().
4610
4611    VALGRIND_MALLOCLIKE_BLOCK should be put immediately after the point where a
4612    heap block -- that will be used by the client program -- is allocated.
4613    It's best to put it at the outermost level of the allocator if possible;
4614    for example, if you have a function my_alloc() which calls
4615    internal_alloc(), and the client request is put inside internal_alloc(),
4616    stack traces relating to the heap block will contain entries for both
4617    my_alloc() and internal_alloc(), which is probably not what you want.
4618
4619    For Memcheck users: if you use VALGRIND_MALLOCLIKE_BLOCK to carve out
4620    custom blocks from within a heap block, B, that has been allocated with
4621    malloc/calloc/new/etc, then block B will be *ignored* during leak-checking
4622    -- the custom blocks will take precedence.
4623
4624    VALGRIND_FREELIKE_BLOCK is the partner to VALGRIND_MALLOCLIKE_BLOCK.  For
4625    Memcheck, it does two things:
4626
4627    - It records that the block has been deallocated.  This assumes that the
4628      block was annotated as having been allocated via
4629      VALGRIND_MALLOCLIKE_BLOCK.  Otherwise, an error will be issued.
4630
4631    - It marks the block as being unaddressable.
4632
4633    VALGRIND_FREELIKE_BLOCK should be put immediately after the point where a
4634    heap block is deallocated.
4635
4636    In many cases, these two client requests will not be enough to get your
4637    allocator working well with Memcheck.  More specifically, if your allocator
4638    writes to freed blocks in any way then a VALGRIND_MAKE_MEM_UNDEFINED call
4639    will be necessary to mark the memory as addressable just before the zeroing
4640    occurs, otherwise you'll get a lot of invalid write errors.  For example,
4641    you'll need to do this if your allocator recycles freed blocks, but it
4642    zeroes them before handing them back out (via VALGRIND_MALLOCLIKE_BLOCK).
4643    Alternatively, if your allocator reuses freed blocks for allocator-internal
4644    data structures, VALGRIND_MAKE_MEM_UNDEFINED calls will also be necessary.
4645
4646    Really, what's happening is a blurring of the lines between the client
4647    program and the allocator... after VALGRIND_FREELIKE_BLOCK is called, the
4648    memory should be considered unaddressable to the client program, but the
4649    allocator knows more than the rest of the client program and so may be able
4650    to safely access it.  Extra client requests are necessary for Valgrind to
4651    understand the distinction between the allocator and the rest of the
4652    program.
4653
4654    Note: there is currently no VALGRIND_REALLOCLIKE_BLOCK client request;  it
4655    has to be emulated with MALLOCLIKE/FREELIKE and memory copying.
4656
4657    Ignored if addr == 0.
4658 */
4659 #define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)    \
4660    {unsigned int _qzz_res;                                        \
4661     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4662                                VG_USERREQ__MALLOCLIKE_BLOCK,      \
4663                                addr, sizeB, rzB, is_zeroed, 0);   \
4664    }
4665
4666 /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
4667    Ignored if addr == 0.
4668 */
4669 #define VALGRIND_FREELIKE_BLOCK(addr, rzB)                        \
4670    {unsigned int _qzz_res;                                        \
4671     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4672                                VG_USERREQ__FREELIKE_BLOCK,        \
4673                                addr, rzB, 0, 0, 0);               \
4674    }
4675
4676 /* Create a memory pool. */
4677 #define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed)             \
4678    {unsigned int _qzz_res;                                        \
4679     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4680                                VG_USERREQ__CREATE_MEMPOOL,        \
4681                                pool, rzB, is_zeroed, 0, 0);       \
4682    }
4683
4684 /* Destroy a memory pool. */
4685 #define VALGRIND_DESTROY_MEMPOOL(pool)                            \
4686    {unsigned int _qzz_res;                                        \
4687     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4688                                VG_USERREQ__DESTROY_MEMPOOL,       \
4689                                pool, 0, 0, 0, 0);                 \
4690    }
4691
4692 /* Associate a piece of memory with a memory pool. */
4693 #define VALGRIND_MEMPOOL_ALLOC(pool, addr, size)                  \
4694    {unsigned int _qzz_res;                                        \
4695     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4696                                VG_USERREQ__MEMPOOL_ALLOC,         \
4697                                pool, addr, size, 0, 0);           \
4698    }
4699
4700 /* Disassociate a piece of memory from a memory pool. */
4701 #define VALGRIND_MEMPOOL_FREE(pool, addr)                         \
4702    {unsigned int _qzz_res;                                        \
4703     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4704                                VG_USERREQ__MEMPOOL_FREE,          \
4705                                pool, addr, 0, 0, 0);              \
4706    }
4707
4708 /* Disassociate any pieces outside a particular range. */
4709 #define VALGRIND_MEMPOOL_TRIM(pool, addr, size)                   \
4710    {unsigned int _qzz_res;                                        \
4711     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4712                                VG_USERREQ__MEMPOOL_TRIM,          \
4713                                pool, addr, size, 0, 0);           \
4714    }
4715
4716 /* Resize and/or move a piece associated with a memory pool. */
4717 #define VALGRIND_MOVE_MEMPOOL(poolA, poolB)                       \
4718    {unsigned int _qzz_res;                                        \
4719     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4720                                VG_USERREQ__MOVE_MEMPOOL,          \
4721                                poolA, poolB, 0, 0, 0);            \
4722    }
4723
4724 /* Resize and/or move a piece associated with a memory pool. */
4725 #define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size)         \
4726    {unsigned int _qzz_res;                                        \
4727     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4728                                VG_USERREQ__MEMPOOL_CHANGE,        \
4729                                pool, addrA, addrB, size, 0);      \
4730    }
4731
4732 /* Return 1 if a mempool exists, else 0. */
4733 #define VALGRIND_MEMPOOL_EXISTS(pool)                             \
4734    __extension__                                                  \
4735    ({unsigned int _qzz_res;                                       \
4736     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4737                                VG_USERREQ__MEMPOOL_EXISTS,        \
4738                                pool, 0, 0, 0, 0);                 \
4739     _qzz_res;                                                     \
4740    })
4741
4742 /* Mark a piece of memory as being a stack. Returns a stack id. */
4743 #define VALGRIND_STACK_REGISTER(start, end)                       \
4744    __extension__                                                  \
4745    ({unsigned int _qzz_res;                                       \
4746     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4747                                VG_USERREQ__STACK_REGISTER,        \
4748                                start, end, 0, 0, 0);              \
4749     _qzz_res;                                                     \
4750    })
4751
4752 /* Unmark the piece of memory associated with a stack id as being a
4753    stack. */
4754 #define VALGRIND_STACK_DEREGISTER(id)                             \
4755    {unsigned int _qzz_res;                                        \
4756     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4757                                VG_USERREQ__STACK_DEREGISTER,      \
4758                                id, 0, 0, 0, 0);                   \
4759    }
4760
4761 /* Change the start and end address of the stack id. */
4762 #define VALGRIND_STACK_CHANGE(id, start, end)                     \
4763    {unsigned int _qzz_res;                                        \
4764     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4765                                VG_USERREQ__STACK_CHANGE,          \
4766                                id, start, end, 0, 0);             \
4767    }
4768
4769 /* Load PDB debug info for Wine PE image_map. */
4770 #define VALGRIND_LOAD_PDB_DEBUGINFO(fd, ptr, total_size, delta)   \
4771    {unsigned int _qzz_res;                                        \
4772     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4773                                VG_USERREQ__LOAD_PDB_DEBUGINFO,    \
4774                                fd, ptr, total_size, delta, 0);    \
4775    }
4776
4777 /* Map a code address to a source file name and line number.  buf64
4778    must point to a 64-byte buffer in the caller's address space.  The
4779    result will be dumped in there and is guaranteed to be zero
4780    terminated.  If no info is found, the first byte is set to zero. */
4781 #define VALGRIND_MAP_IP_TO_SRCLOC(addr, buf64)                    \
4782    {unsigned int _qzz_res;                                        \
4783     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4784                                VG_USERREQ__MAP_IP_TO_SRCLOC,      \
4785                                addr, buf64, 0, 0, 0);             \
4786    }
4787
4788
4789 #undef PLAT_x86_linux
4790 #undef PLAT_amd64_linux
4791 #undef PLAT_ppc32_linux
4792 #undef PLAT_ppc64_linux
4793 #undef PLAT_arm_linux
4794 #undef PLAT_ppc32_aix5
4795 #undef PLAT_ppc64_aix5
4796
4797 #endif   /* __VALGRIND_H */