src/src/valgrind.h

   1 /* -*- c -*-
   2    ----------------------------------------------------------------
   3
   4    Notice that the following BSD-style license applies to this one
   5    file (valgrind.h) only.  The rest of Valgrind is licensed under the
   6    terms of the GNU General Public License, version 2, unless
   7    otherwise indicated.  See the COPYING file in the source
   8    distribution for details.
   9
  10    ----------------------------------------------------------------
  11
  12    This file is part of Valgrind, a dynamic binary instrumentation
  13    framework.
  14
  15    Copyright (C) 2000-2009 Julian Seward.  All rights reserved.
  16
  17    Redistribution and use in source and binary forms, with or without
  18    modification, are permitted provided that the following conditions
  19    are met:
  20
  21    1. Redistributions of source code must retain the above copyright
  22       notice, this list of conditions and the following disclaimer.
  23
  24    2. The origin of this software must not be misrepresented; you must
  25       not claim that you wrote the original software.  If you use this
  26       software in a product, an acknowledgment in the product
  27       documentation would be appreciated but is not required.
  28
  29    3. Altered source versions must be plainly marked as such, and must
  30       not be misrepresented as being the original software.
  31
  32    4. The name of the author may not be used to endorse or promote
  33       products derived from this software without specific prior written
  34       permission.
  35
  36    THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
  37    OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  38    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  39    ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
  40    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  41    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
  42    GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  43    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  44    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  45    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  46    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  47
  48    ----------------------------------------------------------------
  49
  50    Notice that the above BSD-style license applies to this one file
  51    (valgrind.h) only.  The entire rest of Valgrind is licensed under
  52    the terms of the GNU General Public License, version 2.  See the
  53    COPYING file in the source distribution for details.
  54
  55    ----------------------------------------------------------------
  56 */
  57
  58
  59 /* This file is for inclusion into client (your!) code.
  60
  61    You can use these macros to manipulate and query Valgrind's
  62    execution inside your own programs.
  63
  64    The resulting executables will still run without Valgrind, just a
  65    little bit more slowly than they otherwise would, but otherwise
  66    unchanged.  When not running on valgrind, each client request
  67    consumes very few (eg. 7) instructions, so the resulting performance
  68    loss is negligible unless you plan to execute client requests
  69    millions of times per second.  Nevertheless, if that is still a
  70    problem, you can compile with the NVALGRIND symbol defined (gcc
  71    -DNVALGRIND) so that client requests are not even compiled in.  */
  72
  73 #ifndef __VALGRIND_H
  74 #define __VALGRIND_H
  75
  76 #include <stdarg.h>
  77
  78 /* Nb: this file might be included in a file compiled with -ansi.  So
  79    we can't use C++ style "//" comments nor the "asm" keyword (instead
  80    use "__asm__"). */
  81
  82 /* Derive some tags indicating what the target platform is.  Note
  83    that in this file we're using the compiler's CPP symbols for
  84    identifying architectures, which are different to the ones we use
  85    within the rest of Valgrind.  Note, __powerpc__ is active for both
  86    32 and 64-bit PPC, whereas __powerpc64__ is only active for the
  87    latter (on Linux, that is). */
  88 #undef PLAT_x86_linux
  89 #undef PLAT_amd64_linux
  90 #undef PLAT_ppc32_linux
  91 #undef PLAT_ppc64_linux
  92 #undef PLAT_ppc32_aix5
  93 #undef PLAT_ppc64_aix5
  94
  95
  96 #if defined(_AIX) && defined(__64BIT__)
  97 #  define PLAT_ppc64_aix5 1
  98 #elif defined(_AIX) && !defined(__64BIT__)
  99 #  define PLAT_ppc32_aix5 1
 100 #elif defined(__APPLE__) && defined(__i386__)
 101 #  define PLAT_x86_darwin 1
 102 #elif defined(__APPLE__) && defined(__x86_64__)
 103 #  define PLAT_amd64_darwin 1
 104 #elif defined(__i386__)
 105 #  define PLAT_x86_linux 1
 106 #elif defined(__x86_64__)
 107 #  define PLAT_amd64_linux 1
 108 #elif defined(__powerpc__) && !defined(__powerpc64__)
 109 #  define PLAT_ppc32_linux 1
 110 #elif defined(__powerpc__) && defined(__powerpc64__)
 111 #  define PLAT_ppc64_linux 1
 112 #else
 113 /* If we're not compiling for our target platform, don't generate
 114    any inline asms.  */
 115 #  if !defined(NVALGRIND)
 116 #    define NVALGRIND 1
 117 #  endif
 118 #endif
 119
 120
 121 /* ------------------------------------------------------------------ */
 122 /* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS.  There is nothing */
 123 /* in here of use to end-users -- skip to the next section.           */
 124 /* ------------------------------------------------------------------ */
 125
 126 #if defined(NVALGRIND)
 127
 128 /* Define NVALGRIND to completely remove the Valgrind magic sequence
 129    from the compiled code (analogous to NDEBUG's effects on
 130    assert()) */
 131 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 132         _zzq_rlval, _zzq_default, _zzq_request,                   \
 133         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 134    {                                                              \
 135       (_zzq_rlval) = (_zzq_default);                              \
 136    }
 137
 138 #else  /* ! NVALGRIND */
 139
 140 /* The following defines the magic code sequences which the JITter
 141    spots and handles magically.  Don't look too closely at them as
 142    they will rot your brain.
 143
 144    The assembly code sequences for all architectures is in this one
 145    file.  This is because this file must be stand-alone, and we don't
 146    want to have multiple files.
 147
 148    For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default
 149    value gets put in the return slot, so that everything works when
 150    this is executed not under Valgrind.  Args are passed in a memory
 151    block, and so there's no intrinsic limit to the number that could
 152    be passed, but it's currently five.
 153
 154    The macro args are:
 155       _zzq_rlval    result lvalue
 156       _zzq_default  default value (result returned when running on real CPU)
 157       _zzq_request  request code
 158       _zzq_arg1..5  request params
 159
 160    The other two macros are used to support function wrapping, and are
 161    a lot simpler.  VALGRIND_GET_NR_CONTEXT returns the value of the
 162    guest's NRADDR pseudo-register and whatever other information is
 163    needed to safely run the call original from the wrapper: on
 164    ppc64-linux, the R2 value at the divert point is also needed.  This
 165    information is abstracted into a user-visible type, OrigFn.
 166
 167    VALGRIND_CALL_NOREDIR_* behaves the same as the following on the
 168    guest, but guarantees that the branch instruction will not be
 169    redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64:
 170    branch-and-link-to-r11.  VALGRIND_CALL_NOREDIR is just text, not a
 171    complete inline asm, since it needs to be combined with more magic
 172    inline asm stuff to be useful.
 173 */
 174
 175 /* ------------------------- x86-{linux,darwin} ---------------- */
 176
 177 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)
 178
 179 typedef
 180    struct {
 181       unsigned int nraddr; /* where's the code? */
 182    }
 183    OrigFn;
 184
 185 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 186                      "roll $3,  %%edi ; roll $13, %%edi\n\t"      \
 187                      "roll $29, %%edi ; roll $19, %%edi\n\t"
 188
 189 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 190         _zzq_rlval, _zzq_default, _zzq_request,                   \
 191         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 192   { volatile unsigned int _zzq_args[6];                           \
 193     volatile unsigned int _zzq_result;                            \
 194     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 195     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 196     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 197     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 198     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 199     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 200     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 201                      /* %EDX = client_request ( %EAX ) */         \
 202                      "xchgl %%ebx,%%ebx"                          \
 203                      : "=d" (_zzq_result)                         \
 204                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 205                      : "cc", "memory"                             \
 206                     );                                            \
 207     _zzq_rlval = _zzq_result;                                     \
 208   }
 209
 210 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 211   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 212     volatile unsigned int __addr;                                 \
 213     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 214                      /* %EAX = guest_NRADDR */                    \
 215                      "xchgl %%ecx,%%ecx"                          \
 216                      : "=a" (__addr)                              \
 217                      :                                            \
 218                      : "cc", "memory"                             \
 219                     );                                            \
 220     _zzq_orig->nraddr = __addr;                                   \
 221   }
 222
 223 #define VALGRIND_CALL_NOREDIR_EAX                                 \
 224                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 225                      /* call-noredir *%EAX */                     \
 226                      "xchgl %%edx,%%edx\n\t"
 227 #endif /* PLAT_x86_linux || PLAT_x86_darwin */
 228
 229 /* ------------------------ amd64-{linux,darwin} --------------- */
 230
 231 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin)
 232
 233 typedef
 234    struct {
 235       unsigned long long int nraddr; /* where's the code? */
 236    }
 237    OrigFn;
 238
 239 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 240                      "rolq $3,  %%rdi ; rolq $13, %%rdi\n\t"      \
 241                      "rolq $61, %%rdi ; rolq $51, %%rdi\n\t"
 242
 243 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 244         _zzq_rlval, _zzq_default, _zzq_request,                   \
 245         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 246   { volatile unsigned long long int _zzq_args[6];                 \
 247     volatile unsigned long long int _zzq_result;                  \
 248     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 249     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 250     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 251     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 252     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 253     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 254     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 255                      /* %RDX = client_request ( %RAX ) */         \
 256                      "xchgq %%rbx,%%rbx"                          \
 257                      : "=d" (_zzq_result)                         \
 258                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 259                      : "cc", "memory"                             \
 260                     );                                            \
 261     _zzq_rlval = _zzq_result;                                     \
 262   }
 263
 264 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 265   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 266     volatile unsigned long long int __addr;                       \
 267     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 268                      /* %RAX = guest_NRADDR */                    \
 269                      "xchgq %%rcx,%%rcx"                          \
 270                      : "=a" (__addr)                              \
 271                      :                                            \
 272                      : "cc", "memory"                             \
 273                     );                                            \
 274     _zzq_orig->nraddr = __addr;                                   \
 275   }
 276
 277 #define VALGRIND_CALL_NOREDIR_RAX                                 \
 278                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 279                      /* call-noredir *%RAX */                     \
 280                      "xchgq %%rdx,%%rdx\n\t"
 281 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
 282
 283 /* ------------------------ ppc32-linux ------------------------ */
 284
 285 #if defined(PLAT_ppc32_linux)
 286
 287 typedef
 288    struct {
 289       unsigned int nraddr; /* where's the code? */
 290    }
 291    OrigFn;
 292
 293 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 294                      "rlwinm 0,0,3,0,0  ; rlwinm 0,0,13,0,0\n\t"  \
 295                      "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
 296
 297 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 298         _zzq_rlval, _zzq_default, _zzq_request,                   \
 299         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 300                                                                   \
 301   {          unsigned int  _zzq_args[6];                          \
 302              unsigned int  _zzq_result;                           \
 303              unsigned int* _zzq_ptr;                              \
 304     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 305     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 306     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 307     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 308     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 309     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 310     _zzq_ptr = _zzq_args;                                         \
 311     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
 312                      "mr 4,%2\n\t" /*ptr*/                        \
 313                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 314                      /* %R3 = client_request ( %R4 ) */           \
 315                      "or 1,1,1\n\t"                               \
 316                      "mr %0,3"     /*result*/                     \
 317                      : "=b" (_zzq_result)                         \
 318                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
 319                      : "cc", "memory", "r3", "r4");               \
 320     _zzq_rlval = _zzq_result;                                     \
 321   }
 322
 323 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 324   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 325     unsigned int __addr;                                          \
 326     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 327                      /* %R3 = guest_NRADDR */                     \
 328                      "or 2,2,2\n\t"                               \
 329                      "mr %0,3"                                    \
 330                      : "=b" (__addr)                              \
 331                      :                                            \
 332                      : "cc", "memory", "r3"                       \
 333                     );                                            \
 334     _zzq_orig->nraddr = __addr;                                   \
 335   }
 336
 337 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 338                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 339                      /* branch-and-link-to-noredir *%R11 */       \
 340                      "or 3,3,3\n\t"
 341 #endif /* PLAT_ppc32_linux */
 342
 343 /* ------------------------ ppc64-linux ------------------------ */
 344
 345 #if defined(PLAT_ppc64_linux)
 346
 347 typedef
 348    struct {
 349       unsigned long long int nraddr; /* where's the code? */
 350       unsigned long long int r2;  /* what tocptr do we need? */
 351    }
 352    OrigFn;
 353
 354 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 355                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
 356                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
 357
 358 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 359         _zzq_rlval, _zzq_default, _zzq_request,                   \
 360         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 361                                                                   \
 362   {          unsigned long long int  _zzq_args[6];                \
 363     register unsigned long long int  _zzq_result __asm__("r3");   \
 364     register unsigned long long int* _zzq_ptr __asm__("r4");      \
 365     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 366     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 367     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 368     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 369     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 370     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 371     _zzq_ptr = _zzq_args;                                         \
 372     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 373                      /* %R3 = client_request ( %R4 ) */           \
 374                      "or 1,1,1"                                   \
 375                      : "=r" (_zzq_result)                         \
 376                      : "0" (_zzq_default), "r" (_zzq_ptr)         \
 377                      : "cc", "memory");                           \
 378     _zzq_rlval = _zzq_result;                                     \
 379   }
 380
 381 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 382   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 383     register unsigned long long int __addr __asm__("r3");         \
 384     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 385                      /* %R3 = guest_NRADDR */                     \
 386                      "or 2,2,2"                                   \
 387                      : "=r" (__addr)                              \
 388                      :                                            \
 389                      : "cc", "memory"                             \
 390                     );                                            \
 391     _zzq_orig->nraddr = __addr;                                   \
 392     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 393                      /* %R3 = guest_NRADDR_GPR2 */                \
 394                      "or 4,4,4"                                   \
 395                      : "=r" (__addr)                              \
 396                      :                                            \
 397                      : "cc", "memory"                             \
 398                     );                                            \
 399     _zzq_orig->r2 = __addr;                                       \
 400   }
 401
 402 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 403                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 404                      /* branch-and-link-to-noredir *%R11 */       \
 405                      "or 3,3,3\n\t"
 406
 407 #endif /* PLAT_ppc64_linux */
 408
 409 /* ------------------------ ppc32-aix5 ------------------------- */
 410
 411 #if defined(PLAT_ppc32_aix5)
 412
 413 typedef
 414    struct {
 415       unsigned int nraddr; /* where's the code? */
 416       unsigned int r2;  /* what tocptr do we need? */
 417    }
 418    OrigFn;
 419
 420 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 421                      "rlwinm 0,0,3,0,0  ; rlwinm 0,0,13,0,0\n\t"  \
 422                      "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
 423
 424 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 425         _zzq_rlval, _zzq_default, _zzq_request,                   \
 426         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 427                                                                   \
 428   {          unsigned int  _zzq_args[7];                          \
 429     register unsigned int  _zzq_result;                           \
 430     register unsigned int* _zzq_ptr;                              \
 431     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 432     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 433     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 434     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 435     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 436     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 437     _zzq_args[6] = (unsigned int)(_zzq_default);                  \
 438     _zzq_ptr = _zzq_args;                                         \
 439     __asm__ volatile("mr 4,%1\n\t"                                \
 440                      "lwz 3, 24(4)\n\t"                           \
 441                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 442                      /* %R3 = client_request ( %R4 ) */           \
 443                      "or 1,1,1\n\t"                               \
 444                      "mr %0,3"                                    \
 445                      : "=b" (_zzq_result)                         \
 446                      : "b" (_zzq_ptr)                             \
 447                      : "r3", "r4", "cc", "memory");               \
 448     _zzq_rlval = _zzq_result;                                     \
 449   }
 450
 451 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 452   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 453     register unsigned int __addr;                                 \
 454     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 455                      /* %R3 = guest_NRADDR */                     \
 456                      "or 2,2,2\n\t"                               \
 457                      "mr %0,3"                                    \
 458                      : "=b" (__addr)                              \
 459                      :                                            \
 460                      : "r3", "cc", "memory"                       \
 461                     );                                            \
 462     _zzq_orig->nraddr = __addr;                                   \
 463     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 464                      /* %R3 = guest_NRADDR_GPR2 */                \
 465                      "or 4,4,4\n\t"                               \
 466                      "mr %0,3"                                    \
 467                      : "=b" (__addr)                              \
 468                      :                                            \
 469                      : "r3", "cc", "memory"                       \
 470                     );                                            \
 471     _zzq_orig->r2 = __addr;                                       \
 472   }
 473
 474 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 475                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 476                      /* branch-and-link-to-noredir *%R11 */       \
 477                      "or 3,3,3\n\t"
 478
 479 #endif /* PLAT_ppc32_aix5 */
 480
 481 /* ------------------------ ppc64-aix5 ------------------------- */
 482
 483 #if defined(PLAT_ppc64_aix5)
 484
 485 typedef
 486    struct {
 487       unsigned long long int nraddr; /* where's the code? */
 488       unsigned long long int r2;  /* what tocptr do we need? */
 489    }
 490    OrigFn;
 491
 492 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 493                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
 494                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
 495
 496 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 497         _zzq_rlval, _zzq_default, _zzq_request,                   \
 498         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 499                                                                   \
 500   {          unsigned long long int  _zzq_args[7];                \
 501     register unsigned long long int  _zzq_result;                 \
 502     register unsigned long long int* _zzq_ptr;                    \
 503     _zzq_args[0] = (unsigned int long long)(_zzq_request);        \
 504     _zzq_args[1] = (unsigned int long long)(_zzq_arg1);           \
 505     _zzq_args[2] = (unsigned int long long)(_zzq_arg2);           \
 506     _zzq_args[3] = (unsigned int long long)(_zzq_arg3);           \
 507     _zzq_args[4] = (unsigned int long long)(_zzq_arg4);           \
 508     _zzq_args[5] = (unsigned int long long)(_zzq_arg5);           \
 509     _zzq_args[6] = (unsigned int long long)(_zzq_default);        \
 510     _zzq_ptr = _zzq_args;                                         \
 511     __asm__ volatile("mr 4,%1\n\t"                                \
 512                      "ld 3, 48(4)\n\t"                            \
 513                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 514                      /* %R3 = client_request ( %R4 ) */           \
 515                      "or 1,1,1\n\t"                               \
 516                      "mr %0,3"                                    \
 517                      : "=b" (_zzq_result)                         \
 518                      : "b" (_zzq_ptr)                             \
 519                      : "r3", "r4", "cc", "memory");               \
 520     _zzq_rlval = _zzq_result;                                     \
 521   }
 522
 523 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 524   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 525     register unsigned long long int __addr;                       \
 526     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 527                      /* %R3 = guest_NRADDR */                     \
 528                      "or 2,2,2\n\t"                               \
 529                      "mr %0,3"                                    \
 530                      : "=b" (__addr)                              \
 531                      :                                            \
 532                      : "r3", "cc", "memory"                       \
 533                     );                                            \
 534     _zzq_orig->nraddr = __addr;                                   \
 535     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 536                      /* %R3 = guest_NRADDR_GPR2 */                \
 537                      "or 4,4,4\n\t"                               \
 538                      "mr %0,3"                                    \
 539                      : "=b" (__addr)                              \
 540                      :                                            \
 541                      : "r3", "cc", "memory"                       \
 542                     );                                            \
 543     _zzq_orig->r2 = __addr;                                       \
 544   }
 545
 546 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 547                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 548                      /* branch-and-link-to-noredir *%R11 */       \
 549                      "or 3,3,3\n\t"
 550
 551 #endif /* PLAT_ppc64_aix5 */
 552
 553 /* Insert assembly code for other platforms here... */
 554
 555 #endif /* NVALGRIND */
 556
 557
 558 /* ------------------------------------------------------------------ */
 559 /* PLATFORM SPECIFICS for FUNCTION WRAPPING.  This is all very        */
 560 /* ugly.  It's the least-worst tradeoff I can think of.               */
 561 /* ------------------------------------------------------------------ */
 562
 563 /* This section defines magic (a.k.a appalling-hack) macros for doing
 564    guaranteed-no-redirection macros, so as to get from function
 565    wrappers to the functions they are wrapping.  The whole point is to
 566    construct standard call sequences, but to do the call itself with a
 567    special no-redirect call pseudo-instruction that the JIT
 568    understands and handles specially.  This section is long and
 569    repetitious, and I can't see a way to make it shorter.
 570
 571    The naming scheme is as follows:
 572
 573       CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc}
 574
 575    'W' stands for "word" and 'v' for "void".  Hence there are
 576    different macros for calling arity 0, 1, 2, 3, 4, etc, functions,
 577    and for each, the possibility of returning a word-typed result, or
 578    no result.
 579 */
 580
 581 /* Use these to write the name of your wrapper.  NOTE: duplicates
 582    VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h. */
 583
 584 /* Use an extra level of macroisation so as to ensure the soname/fnname
 585    args are fully macro-expanded before pasting them together. */
 586 #define VG_CONCAT4(_aa,_bb,_cc,_dd) _aa##_bb##_cc##_dd
 587
 588 #define I_WRAP_SONAME_FNNAME_ZU(soname,fnname)                    \
 589    VG_CONCAT4(_vgwZU_,soname,_,fnname)
 590
 591 #define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname)                    \
 592    VG_CONCAT4(_vgwZZ_,soname,_,fnname)
 593
 594 /* Use this macro from within a wrapper function to collect the
 595    context (address and possibly other info) of the original function.
 596    Once you have that you can then use it in one of the CALL_FN_
 597    macros.  The type of the argument _lval is OrigFn. */
 598 #define VALGRIND_GET_ORIG_FN(_lval)  VALGRIND_GET_NR_CONTEXT(_lval)
 599
 600 /* Derivatives of the main macros below, for calling functions
 601    returning void. */
 602
 603 #define CALL_FN_v_v(fnptr)                                        \
 604    do { volatile unsigned long _junk;                             \
 605         CALL_FN_W_v(_junk,fnptr); } while (0)
 606
 607 #define CALL_FN_v_W(fnptr, arg1)                                  \
 608    do { volatile unsigned long _junk;                             \
 609         CALL_FN_W_W(_junk,fnptr,arg1); } while (0)
 610
 611 #define CALL_FN_v_WW(fnptr, arg1,arg2)                            \
 612    do { volatile unsigned long _junk;                             \
 613         CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0)
 614
 615 #define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3)                      \
 616    do { volatile unsigned long _junk;                             \
 617         CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0)
 618
 619 #define CALL_FN_v_WWWW(fnptr, arg1,arg2,arg3,arg4)                \
 620    do { volatile unsigned long _junk;                             \
 621         CALL_FN_W_WWWW(_junk,fnptr,arg1,arg2,arg3,arg4); } while (0)
 622
 623 #define CALL_FN_v_5W(fnptr, arg1,arg2,arg3,arg4,arg5)             \
 624    do { volatile unsigned long _junk;                             \
 625         CALL_FN_W_5W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5); } while (0)
 626
 627 #define CALL_FN_v_6W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6)        \
 628    do { volatile unsigned long _junk;                             \
 629         CALL_FN_W_6W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6); } while (0)
 630
 631 #define CALL_FN_v_7W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6,arg7)   \
 632    do { volatile unsigned long _junk;                             \
 633         CALL_FN_W_7W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6,arg7); } while (0)
 634
 635 /* ------------------------- x86-{linux,darwin} ---------------- */
 636
 637 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)
 638
 639 /* These regs are trashed by the hidden call.  No need to mention eax
 640    as gcc can already see that, plus causes gcc to bomb. */
 641 #define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx"
 642
 643 /* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned
 644    long) == 4. */
 645
 646 #define CALL_FN_W_v(lval, orig)                                   \
 647    do {                                                           \
 648       volatile OrigFn        _orig = (orig);                      \
 649       volatile unsigned long _argvec[1];                          \
 650       volatile unsigned long _res;                                \
 651       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 652       __asm__ volatile(                                           \
 653          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 654          VALGRIND_CALL_NOREDIR_EAX                                \
 655          : /*out*/   "=a" (_res)                                  \
 656          : /*in*/    "a" (&_argvec[0])                            \
 657          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 658       );                                                          \
 659       lval = (__typeof__(lval)) _res;                             \
 660    } while (0)
 661
 662 #define CALL_FN_W_W(lval, orig, arg1)                             \
 663    do {                                                           \
 664       volatile OrigFn        _orig = (orig);                      \
 665       volatile unsigned long _argvec[2];                          \
 666       volatile unsigned long _res;                                \
 667       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 668       _argvec[1] = (unsigned long)(arg1);                         \
 669       __asm__ volatile(                                           \
 670          "pushl 4(%%eax)\n\t"                                     \
 671          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 672          VALGRIND_CALL_NOREDIR_EAX                                \
 673          "addl $4, %%esp\n"                                       \
 674          : /*out*/   "=a" (_res)                                  \
 675          : /*in*/    "a" (&_argvec[0])                            \
 676          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 677       );                                                          \
 678       lval = (__typeof__(lval)) _res;                             \
 679    } while (0)
 680
 681 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
 682    do {                                                           \
 683       volatile OrigFn        _orig = (orig);                      \
 684       volatile unsigned long _argvec[3];                          \
 685       volatile unsigned long _res;                                \
 686       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 687       _argvec[1] = (unsigned long)(arg1);                         \
 688       _argvec[2] = (unsigned long)(arg2);                         \
 689       __asm__ volatile(                                           \
 690          "pushl 8(%%eax)\n\t"                                     \
 691          "pushl 4(%%eax)\n\t"                                     \
 692          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 693          VALGRIND_CALL_NOREDIR_EAX                                \
 694          "addl $8, %%esp\n"                                       \
 695          : /*out*/   "=a" (_res)                                  \
 696          : /*in*/    "a" (&_argvec[0])                            \
 697          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 698       );                                                          \
 699       lval = (__typeof__(lval)) _res;                             \
 700    } while (0)
 701
 702 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
 703    do {                                                           \
 704       volatile OrigFn        _orig = (orig);                      \
 705       volatile unsigned long _argvec[4];                          \
 706       volatile unsigned long _res;                                \
 707       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 708       _argvec[1] = (unsigned long)(arg1);                         \
 709       _argvec[2] = (unsigned long)(arg2);                         \
 710       _argvec[3] = (unsigned long)(arg3);                         \
 711       __asm__ volatile(                                           \
 712          "pushl 12(%%eax)\n\t"                                    \
 713          "pushl 8(%%eax)\n\t"                                     \
 714          "pushl 4(%%eax)\n\t"                                     \
 715          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 716          VALGRIND_CALL_NOREDIR_EAX                                \
 717          "addl $12, %%esp\n"                                      \
 718          : /*out*/   "=a" (_res)                                  \
 719          : /*in*/    "a" (&_argvec[0])                            \
 720          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 721       );                                                          \
 722       lval = (__typeof__(lval)) _res;                             \
 723    } while (0)
 724
 725 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
 726    do {                                                           \
 727       volatile OrigFn        _orig = (orig);                      \
 728       volatile unsigned long _argvec[5];                          \
 729       volatile unsigned long _res;                                \
 730       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 731       _argvec[1] = (unsigned long)(arg1);                         \
 732       _argvec[2] = (unsigned long)(arg2);                         \
 733       _argvec[3] = (unsigned long)(arg3);                         \
 734       _argvec[4] = (unsigned long)(arg4);                         \
 735       __asm__ volatile(                                           \
 736          "pushl 16(%%eax)\n\t"                                    \
 737          "pushl 12(%%eax)\n\t"                                    \
 738          "pushl 8(%%eax)\n\t"                                     \
 739          "pushl 4(%%eax)\n\t"                                     \
 740          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 741          VALGRIND_CALL_NOREDIR_EAX                                \
 742          "addl $16, %%esp\n"                                      \
 743          : /*out*/   "=a" (_res)                                  \
 744          : /*in*/    "a" (&_argvec[0])                            \
 745          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 746       );                                                          \
 747       lval = (__typeof__(lval)) _res;                             \
 748    } while (0)
 749
 750 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
 751    do {                                                           \
 752       volatile OrigFn        _orig = (orig);                      \
 753       volatile unsigned long _argvec[6];                          \
 754       volatile unsigned long _res;                                \
 755       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 756       _argvec[1] = (unsigned long)(arg1);                         \
 757       _argvec[2] = (unsigned long)(arg2);                         \
 758       _argvec[3] = (unsigned long)(arg3);                         \
 759       _argvec[4] = (unsigned long)(arg4);                         \
 760       _argvec[5] = (unsigned long)(arg5);                         \
 761       __asm__ volatile(                                           \
 762          "pushl 20(%%eax)\n\t"                                    \
 763          "pushl 16(%%eax)\n\t"                                    \
 764          "pushl 12(%%eax)\n\t"                                    \
 765          "pushl 8(%%eax)\n\t"                                     \
 766          "pushl 4(%%eax)\n\t"                                     \
 767          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 768          VALGRIND_CALL_NOREDIR_EAX                                \
 769          "addl $20, %%esp\n"                                      \
 770          : /*out*/   "=a" (_res)                                  \
 771          : /*in*/    "a" (&_argvec[0])                            \
 772          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 773       );                                                          \
 774       lval = (__typeof__(lval)) _res;                             \
 775    } while (0)
 776
 777 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
 778    do {                                                           \
 779       volatile OrigFn        _orig = (orig);                      \
 780       volatile unsigned long _argvec[7];                          \
 781       volatile unsigned long _res;                                \
 782       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 783       _argvec[1] = (unsigned long)(arg1);                         \
 784       _argvec[2] = (unsigned long)(arg2);                         \
 785       _argvec[3] = (unsigned long)(arg3);                         \
 786       _argvec[4] = (unsigned long)(arg4);                         \
 787       _argvec[5] = (unsigned long)(arg5);                         \
 788       _argvec[6] = (unsigned long)(arg6);                         \
 789       __asm__ volatile(                                           \
 790          "pushl 24(%%eax)\n\t"                                    \
 791          "pushl 20(%%eax)\n\t"                                    \
 792          "pushl 16(%%eax)\n\t"                                    \
 793          "pushl 12(%%eax)\n\t"                                    \
 794          "pushl 8(%%eax)\n\t"                                     \
 795          "pushl 4(%%eax)\n\t"                                     \
 796          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 797          VALGRIND_CALL_NOREDIR_EAX                                \
 798          "addl $24, %%esp\n"                                      \
 799          : /*out*/   "=a" (_res)                                  \
 800          : /*in*/    "a" (&_argvec[0])                            \
 801          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 802       );                                                          \
 803       lval = (__typeof__(lval)) _res;                             \
 804    } while (0)
 805
 806 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
 807                                  arg7)                            \
 808    do {                                                           \
 809       volatile OrigFn        _orig = (orig);                      \
 810       volatile unsigned long _argvec[8];                          \
 811       volatile unsigned long _res;                                \
 812       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 813       _argvec[1] = (unsigned long)(arg1);                         \
 814       _argvec[2] = (unsigned long)(arg2);                         \
 815       _argvec[3] = (unsigned long)(arg3);                         \
 816       _argvec[4] = (unsigned long)(arg4);                         \
 817       _argvec[5] = (unsigned long)(arg5);                         \
 818       _argvec[6] = (unsigned long)(arg6);                         \
 819       _argvec[7] = (unsigned long)(arg7);                         \
 820       __asm__ volatile(                                           \
 821          "pushl 28(%%eax)\n\t"                                    \
 822          "pushl 24(%%eax)\n\t"                                    \
 823          "pushl 20(%%eax)\n\t"                                    \
 824          "pushl 16(%%eax)\n\t"                                    \
 825          "pushl 12(%%eax)\n\t"                                    \
 826          "pushl 8(%%eax)\n\t"                                     \
 827          "pushl 4(%%eax)\n\t"                                     \
 828          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 829          VALGRIND_CALL_NOREDIR_EAX                                \
 830          "addl $28, %%esp\n"                                      \
 831          : /*out*/   "=a" (_res)                                  \
 832          : /*in*/    "a" (&_argvec[0])                            \
 833          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 834       );                                                          \
 835       lval = (__typeof__(lval)) _res;                             \
 836    } while (0)
 837
 838 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
 839                                  arg7,arg8)                       \
 840    do {                                                           \
 841       volatile OrigFn        _orig = (orig);                      \
 842       volatile unsigned long _argvec[9];                          \
 843       volatile unsigned long _res;                                \
 844       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 845       _argvec[1] = (unsigned long)(arg1);                         \
 846       _argvec[2] = (unsigned long)(arg2);                         \
 847       _argvec[3] = (unsigned long)(arg3);                         \
 848       _argvec[4] = (unsigned long)(arg4);                         \
 849       _argvec[5] = (unsigned long)(arg5);                         \
 850       _argvec[6] = (unsigned long)(arg6);                         \
 851       _argvec[7] = (unsigned long)(arg7);                         \
 852       _argvec[8] = (unsigned long)(arg8);                         \
 853       __asm__ volatile(                                           \
 854          "pushl 32(%%eax)\n\t"                                    \
 855          "pushl 28(%%eax)\n\t"                                    \
 856          "pushl 24(%%eax)\n\t"                                    \
 857          "pushl 20(%%eax)\n\t"                                    \
 858          "pushl 16(%%eax)\n\t"                                    \
 859          "pushl 12(%%eax)\n\t"                                    \
 860          "pushl 8(%%eax)\n\t"                                     \
 861          "pushl 4(%%eax)\n\t"                                     \
 862          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 863          VALGRIND_CALL_NOREDIR_EAX                                \
 864          "addl $32, %%esp\n"                                      \
 865          : /*out*/   "=a" (_res)                                  \
 866          : /*in*/    "a" (&_argvec[0])                            \
 867          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 868       );                                                          \
 869       lval = (__typeof__(lval)) _res;                             \
 870    } while (0)
 871
 872 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
 873                                  arg7,arg8,arg9)                  \
 874    do {                                                           \
 875       volatile OrigFn        _orig = (orig);                      \
 876       volatile unsigned long _argvec[10];                         \
 877       volatile unsigned long _res;                                \
 878       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 879       _argvec[1] = (unsigned long)(arg1);                         \
 880       _argvec[2] = (unsigned long)(arg2);                         \
 881       _argvec[3] = (unsigned long)(arg3);                         \
 882       _argvec[4] = (unsigned long)(arg4);                         \
 883       _argvec[5] = (unsigned long)(arg5);                         \
 884       _argvec[6] = (unsigned long)(arg6);                         \
 885       _argvec[7] = (unsigned long)(arg7);                         \
 886       _argvec[8] = (unsigned long)(arg8);                         \
 887       _argvec[9] = (unsigned long)(arg9);                         \
 888       __asm__ volatile(                                           \
 889          "pushl 36(%%eax)\n\t"                                    \
 890          "pushl 32(%%eax)\n\t"                                    \
 891          "pushl 28(%%eax)\n\t"                                    \
 892          "pushl 24(%%eax)\n\t"                                    \
 893          "pushl 20(%%eax)\n\t"                                    \
 894          "pushl 16(%%eax)\n\t"                                    \
 895          "pushl 12(%%eax)\n\t"                                    \
 896          "pushl 8(%%eax)\n\t"                                     \
 897          "pushl 4(%%eax)\n\t"                                     \
 898          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 899          VALGRIND_CALL_NOREDIR_EAX                                \
 900          "addl $36, %%esp\n"                                      \
 901          : /*out*/   "=a" (_res)                                  \
 902          : /*in*/    "a" (&_argvec[0])                            \
 903          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 904       );                                                          \
 905       lval = (__typeof__(lval)) _res;                             \
 906    } while (0)
 907
 908 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
 909                                   arg7,arg8,arg9,arg10)           \
 910    do {                                                           \
 911       volatile OrigFn        _orig = (orig);                      \
 912       volatile unsigned long _argvec[11];                         \
 913       volatile unsigned long _res;                                \
 914       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 915       _argvec[1] = (unsigned long)(arg1);                         \
 916       _argvec[2] = (unsigned long)(arg2);                         \
 917       _argvec[3] = (unsigned long)(arg3);                         \
 918       _argvec[4] = (unsigned long)(arg4);                         \
 919       _argvec[5] = (unsigned long)(arg5);                         \
 920       _argvec[6] = (unsigned long)(arg6);                         \
 921       _argvec[7] = (unsigned long)(arg7);                         \
 922       _argvec[8] = (unsigned long)(arg8);                         \
 923       _argvec[9] = (unsigned long)(arg9);                         \
 924       _argvec[10] = (unsigned long)(arg10);                       \
 925       __asm__ volatile(                                           \
 926          "pushl 40(%%eax)\n\t"                                    \
 927          "pushl 36(%%eax)\n\t"                                    \
 928          "pushl 32(%%eax)\n\t"                                    \
 929          "pushl 28(%%eax)\n\t"                                    \
 930          "pushl 24(%%eax)\n\t"                                    \
 931          "pushl 20(%%eax)\n\t"                                    \
 932          "pushl 16(%%eax)\n\t"                                    \
 933          "pushl 12(%%eax)\n\t"                                    \
 934          "pushl 8(%%eax)\n\t"                                     \
 935          "pushl 4(%%eax)\n\t"                                     \
 936          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 937          VALGRIND_CALL_NOREDIR_EAX                                \
 938          "addl $40, %%esp\n"                                      \
 939          : /*out*/   "=a" (_res)                                  \
 940          : /*in*/    "a" (&_argvec[0])                            \
 941          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 942       );                                                          \
 943       lval = (__typeof__(lval)) _res;                             \
 944    } while (0)
 945
 946 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
 947                                   arg6,arg7,arg8,arg9,arg10,      \
 948                                   arg11)                          \
 949    do {                                                           \
 950       volatile OrigFn        _orig = (orig);                      \
 951       volatile unsigned long _argvec[12];                         \
 952       volatile unsigned long _res;                                \
 953       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 954       _argvec[1] = (unsigned long)(arg1);                         \
 955       _argvec[2] = (unsigned long)(arg2);                         \
 956       _argvec[3] = (unsigned long)(arg3);                         \
 957       _argvec[4] = (unsigned long)(arg4);                         \
 958       _argvec[5] = (unsigned long)(arg5);                         \
 959       _argvec[6] = (unsigned long)(arg6);                         \
 960       _argvec[7] = (unsigned long)(arg7);                         \
 961       _argvec[8] = (unsigned long)(arg8);                         \
 962       _argvec[9] = (unsigned long)(arg9);                         \
 963       _argvec[10] = (unsigned long)(arg10);                       \
 964       _argvec[11] = (unsigned long)(arg11);                       \
 965       __asm__ volatile(                                           \
 966          "pushl 44(%%eax)\n\t"                                    \
 967          "pushl 40(%%eax)\n\t"                                    \
 968          "pushl 36(%%eax)\n\t"                                    \
 969          "pushl 32(%%eax)\n\t"                                    \
 970          "pushl 28(%%eax)\n\t"                                    \
 971          "pushl 24(%%eax)\n\t"                                    \
 972          "pushl 20(%%eax)\n\t"                                    \
 973          "pushl 16(%%eax)\n\t"                                    \
 974          "pushl 12(%%eax)\n\t"                                    \
 975          "pushl 8(%%eax)\n\t"                                     \
 976          "pushl 4(%%eax)\n\t"                                     \
 977          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 978          VALGRIND_CALL_NOREDIR_EAX                                \
 979          "addl $44, %%esp\n"                                      \
 980          : /*out*/   "=a" (_res)                                  \
 981          : /*in*/    "a" (&_argvec[0])                            \
 982          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 983       );                                                          \
 984       lval = (__typeof__(lval)) _res;                             \
 985    } while (0)
 986
 987 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
 988                                   arg6,arg7,arg8,arg9,arg10,      \
 989                                   arg11,arg12)                    \
 990    do {                                                           \
 991       volatile OrigFn        _orig = (orig);                      \
 992       volatile unsigned long _argvec[13];                         \
 993       volatile unsigned long _res;                                \
 994       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 995       _argvec[1] = (unsigned long)(arg1);                         \
 996       _argvec[2] = (unsigned long)(arg2);                         \
 997       _argvec[3] = (unsigned long)(arg3);                         \
 998       _argvec[4] = (unsigned long)(arg4);                         \
 999       _argvec[5] = (unsigned long)(arg5);                         \
1000       _argvec[6] = (unsigned long)(arg6);                         \
1001       _argvec[7] = (unsigned long)(arg7);                         \
1002       _argvec[8] = (unsigned long)(arg8);                         \
1003       _argvec[9] = (unsigned long)(arg9);                         \
1004       _argvec[10] = (unsigned long)(arg10);                       \
1005       _argvec[11] = (unsigned long)(arg11);                       \
1006       _argvec[12] = (unsigned long)(arg12);                       \
1007       __asm__ volatile(                                           \
1008          "pushl 48(%%eax)\n\t"                                    \
1009          "pushl 44(%%eax)\n\t"                                    \
1010          "pushl 40(%%eax)\n\t"                                    \
1011          "pushl 36(%%eax)\n\t"                                    \
1012          "pushl 32(%%eax)\n\t"                                    \
1013          "pushl 28(%%eax)\n\t"                                    \
1014          "pushl 24(%%eax)\n\t"                                    \
1015          "pushl 20(%%eax)\n\t"                                    \
1016          "pushl 16(%%eax)\n\t"                                    \
1017          "pushl 12(%%eax)\n\t"                                    \
1018          "pushl 8(%%eax)\n\t"                                     \
1019          "pushl 4(%%eax)\n\t"                                     \
1020          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1021          VALGRIND_CALL_NOREDIR_EAX                                \
1022          "addl $48, %%esp\n"                                      \
1023          : /*out*/   "=a" (_res)                                  \
1024          : /*in*/    "a" (&_argvec[0])                            \
1025          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1026       );                                                          \
1027       lval = (__typeof__(lval)) _res;                             \
1028    } while (0)
1029
1030 #endif /* PLAT_x86_linux || PLAT_x86_darwin */
1031
1032 /* ------------------------ amd64-{linux,darwin} --------------- */
1033
1034 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin)
1035
1036 /* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */
1037
1038 /* These regs are trashed by the hidden call. */
1039 #define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi",       \
1040                             "rdi", "r8", "r9", "r10", "r11"
1041
1042 /* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned
1043    long) == 8. */
1044
1045 /* NB 9 Sept 07.  There is a nasty kludge here in all these CALL_FN_
1046    macros.  In order not to trash the stack redzone, we need to drop
1047    %rsp by 128 before the hidden call, and restore afterwards.  The
1048    nastyness is that it is only by luck that the stack still appears
1049    to be unwindable during the hidden call - since then the behaviour
1050    of any routine using this macro does not match what the CFI data
1051    says.  Sigh.
1052
1053    Why is this important?  Imagine that a wrapper has a stack
1054    allocated local, and passes to the hidden call, a pointer to it.
1055    Because gcc does not know about the hidden call, it may allocate
1056    that local in the redzone.  Unfortunately the hidden call may then
1057    trash it before it comes to use it.  So we must step clear of the
1058    redzone, for the duration of the hidden call, to make it safe.
1059
1060    Probably the same problem afflicts the other redzone-style ABIs too
1061    (ppc64-linux, ppc32-aix5, ppc64-aix5); but for those, the stack is
1062    self describing (none of this CFI nonsense) so at least messing
1063    with the stack pointer doesn't give a danger of non-unwindable
1064    stack. */
1065
1066 #define CALL_FN_W_v(lval, orig)                                   \
1067    do {                                                           \
1068       volatile OrigFn        _orig = (orig);                      \
1069       volatile unsigned long _argvec[1];                          \
1070       volatile unsigned long _res;                                \
1071       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1072       __asm__ volatile(                                           \
1073          "subq $128,%%rsp\n\t"                                    \
1074          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1075          VALGRIND_CALL_NOREDIR_RAX                                \
1076          "addq $128,%%rsp\n\t"                                    \
1077          : /*out*/   "=a" (_res)                                  \
1078          : /*in*/    "a" (&_argvec[0])                            \
1079          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1080       );                                                          \
1081       lval = (__typeof__(lval)) _res;                             \
1082    } while (0)
1083
1084 #define CALL_FN_W_W(lval, orig, arg1)                             \
1085    do {                                                           \
1086       volatile OrigFn        _orig = (orig);                      \
1087       volatile unsigned long _argvec[2];                          \
1088       volatile unsigned long _res;                                \
1089       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1090       _argvec[1] = (unsigned long)(arg1);                         \
1091       __asm__ volatile(                                           \
1092          "subq $128,%%rsp\n\t"                                    \
1093          "movq 8(%%rax), %%rdi\n\t"                               \
1094          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1095          VALGRIND_CALL_NOREDIR_RAX                                \
1096          "addq $128,%%rsp\n\t"                                    \
1097          : /*out*/   "=a" (_res)                                  \
1098          : /*in*/    "a" (&_argvec[0])                            \
1099          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1100       );                                                          \
1101       lval = (__typeof__(lval)) _res;                             \
1102    } while (0)
1103
1104 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1105    do {                                                           \
1106       volatile OrigFn        _orig = (orig);                      \
1107       volatile unsigned long _argvec[3];                          \
1108       volatile unsigned long _res;                                \
1109       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1110       _argvec[1] = (unsigned long)(arg1);                         \
1111       _argvec[2] = (unsigned long)(arg2);                         \
1112       __asm__ volatile(                                           \
1113          "subq $128,%%rsp\n\t"                                    \
1114          "movq 16(%%rax), %%rsi\n\t"                              \
1115          "movq 8(%%rax), %%rdi\n\t"                               \
1116          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1117          VALGRIND_CALL_NOREDIR_RAX                                \
1118          "addq $128,%%rsp\n\t"                                    \
1119          : /*out*/   "=a" (_res)                                  \
1120          : /*in*/    "a" (&_argvec[0])                            \
1121          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1122       );                                                          \
1123       lval = (__typeof__(lval)) _res;                             \
1124    } while (0)
1125
1126 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1127    do {                                                           \
1128       volatile OrigFn        _orig = (orig);                      \
1129       volatile unsigned long _argvec[4];                          \
1130       volatile unsigned long _res;                                \
1131       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1132       _argvec[1] = (unsigned long)(arg1);                         \
1133       _argvec[2] = (unsigned long)(arg2);                         \
1134       _argvec[3] = (unsigned long)(arg3);                         \
1135       __asm__ volatile(                                           \
1136          "subq $128,%%rsp\n\t"                                    \
1137          "movq 24(%%rax), %%rdx\n\t"                              \
1138          "movq 16(%%rax), %%rsi\n\t"                              \
1139          "movq 8(%%rax), %%rdi\n\t"                               \
1140          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1141          VALGRIND_CALL_NOREDIR_RAX                                \
1142          "addq $128,%%rsp\n\t"                                    \
1143          : /*out*/   "=a" (_res)                                  \
1144          : /*in*/    "a" (&_argvec[0])                            \
1145          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1146       );                                                          \
1147       lval = (__typeof__(lval)) _res;                             \
1148    } while (0)
1149
1150 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1151    do {                                                           \
1152       volatile OrigFn        _orig = (orig);                      \
1153       volatile unsigned long _argvec[5];                          \
1154       volatile unsigned long _res;                                \
1155       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1156       _argvec[1] = (unsigned long)(arg1);                         \
1157       _argvec[2] = (unsigned long)(arg2);                         \
1158       _argvec[3] = (unsigned long)(arg3);                         \
1159       _argvec[4] = (unsigned long)(arg4);                         \
1160       __asm__ volatile(                                           \
1161          "subq $128,%%rsp\n\t"                                    \
1162          "movq 32(%%rax), %%rcx\n\t"                              \
1163          "movq 24(%%rax), %%rdx\n\t"                              \
1164          "movq 16(%%rax), %%rsi\n\t"                              \
1165          "movq 8(%%rax), %%rdi\n\t"                               \
1166          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1167          VALGRIND_CALL_NOREDIR_RAX                                \
1168          "addq $128,%%rsp\n\t"                                    \
1169          : /*out*/   "=a" (_res)                                  \
1170          : /*in*/    "a" (&_argvec[0])                            \
1171          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1172       );                                                          \
1173       lval = (__typeof__(lval)) _res;                             \
1174    } while (0)
1175
1176 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1177    do {                                                           \
1178       volatile OrigFn        _orig = (orig);                      \
1179       volatile unsigned long _argvec[6];                          \
1180       volatile unsigned long _res;                                \
1181       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1182       _argvec[1] = (unsigned long)(arg1);                         \
1183       _argvec[2] = (unsigned long)(arg2);                         \
1184       _argvec[3] = (unsigned long)(arg3);                         \
1185       _argvec[4] = (unsigned long)(arg4);                         \
1186       _argvec[5] = (unsigned long)(arg5);                         \
1187       __asm__ volatile(                                           \
1188          "subq $128,%%rsp\n\t"                                    \
1189          "movq 40(%%rax), %%r8\n\t"                               \
1190          "movq 32(%%rax), %%rcx\n\t"                              \
1191          "movq 24(%%rax), %%rdx\n\t"                              \
1192          "movq 16(%%rax), %%rsi\n\t"                              \
1193          "movq 8(%%rax), %%rdi\n\t"                               \
1194          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1195          VALGRIND_CALL_NOREDIR_RAX                                \
1196          "addq $128,%%rsp\n\t"                                    \
1197          : /*out*/   "=a" (_res)                                  \
1198          : /*in*/    "a" (&_argvec[0])                            \
1199          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1200       );                                                          \
1201       lval = (__typeof__(lval)) _res;                             \
1202    } while (0)
1203
1204 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1205    do {                                                           \
1206       volatile OrigFn        _orig = (orig);                      \
1207       volatile unsigned long _argvec[7];                          \
1208       volatile unsigned long _res;                                \
1209       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1210       _argvec[1] = (unsigned long)(arg1);                         \
1211       _argvec[2] = (unsigned long)(arg2);                         \
1212       _argvec[3] = (unsigned long)(arg3);                         \
1213       _argvec[4] = (unsigned long)(arg4);                         \
1214       _argvec[5] = (unsigned long)(arg5);                         \
1215       _argvec[6] = (unsigned long)(arg6);                         \
1216       __asm__ volatile(                                           \
1217          "subq $128,%%rsp\n\t"                                    \
1218          "movq 48(%%rax), %%r9\n\t"                               \
1219          "movq 40(%%rax), %%r8\n\t"                               \
1220          "movq 32(%%rax), %%rcx\n\t"                              \
1221          "movq 24(%%rax), %%rdx\n\t"                              \
1222          "movq 16(%%rax), %%rsi\n\t"                              \
1223          "movq 8(%%rax), %%rdi\n\t"                               \
1224          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1225          "addq $128,%%rsp\n\t"                                    \
1226          VALGRIND_CALL_NOREDIR_RAX                                \
1227          : /*out*/   "=a" (_res)                                  \
1228          : /*in*/    "a" (&_argvec[0])                            \
1229          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1230       );                                                          \
1231       lval = (__typeof__(lval)) _res;                             \
1232    } while (0)
1233
1234 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1235                                  arg7)                            \
1236    do {                                                           \
1237       volatile OrigFn        _orig = (orig);                      \
1238       volatile unsigned long _argvec[8];                          \
1239       volatile unsigned long _res;                                \
1240       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1241       _argvec[1] = (unsigned long)(arg1);                         \
1242       _argvec[2] = (unsigned long)(arg2);                         \
1243       _argvec[3] = (unsigned long)(arg3);                         \
1244       _argvec[4] = (unsigned long)(arg4);                         \
1245       _argvec[5] = (unsigned long)(arg5);                         \
1246       _argvec[6] = (unsigned long)(arg6);                         \
1247       _argvec[7] = (unsigned long)(arg7);                         \
1248       __asm__ volatile(                                           \
1249          "subq $128,%%rsp\n\t"                                    \
1250          "pushq 56(%%rax)\n\t"                                    \
1251          "movq 48(%%rax), %%r9\n\t"                               \
1252          "movq 40(%%rax), %%r8\n\t"                               \
1253          "movq 32(%%rax), %%rcx\n\t"                              \
1254          "movq 24(%%rax), %%rdx\n\t"                              \
1255          "movq 16(%%rax), %%rsi\n\t"                              \
1256          "movq 8(%%rax), %%rdi\n\t"                               \
1257          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1258          VALGRIND_CALL_NOREDIR_RAX                                \
1259          "addq $8, %%rsp\n"                                       \
1260          "addq $128,%%rsp\n\t"                                    \
1261          : /*out*/   "=a" (_res)                                  \
1262          : /*in*/    "a" (&_argvec[0])                            \
1263          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1264       );                                                          \
1265       lval = (__typeof__(lval)) _res;                             \
1266    } while (0)
1267
1268 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1269                                  arg7,arg8)                       \
1270    do {                                                           \
1271       volatile OrigFn        _orig = (orig);                      \
1272       volatile unsigned long _argvec[9];                          \
1273       volatile unsigned long _res;                                \
1274       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1275       _argvec[1] = (unsigned long)(arg1);                         \
1276       _argvec[2] = (unsigned long)(arg2);                         \
1277       _argvec[3] = (unsigned long)(arg3);                         \
1278       _argvec[4] = (unsigned long)(arg4);                         \
1279       _argvec[5] = (unsigned long)(arg5);                         \
1280       _argvec[6] = (unsigned long)(arg6);                         \
1281       _argvec[7] = (unsigned long)(arg7);                         \
1282       _argvec[8] = (unsigned long)(arg8);                         \
1283       __asm__ volatile(                                           \
1284          "subq $128,%%rsp\n\t"                                    \
1285          "pushq 64(%%rax)\n\t"                                    \
1286          "pushq 56(%%rax)\n\t"                                    \
1287          "movq 48(%%rax), %%r9\n\t"                               \
1288          "movq 40(%%rax), %%r8\n\t"                               \
1289          "movq 32(%%rax), %%rcx\n\t"                              \
1290          "movq 24(%%rax), %%rdx\n\t"                              \
1291          "movq 16(%%rax), %%rsi\n\t"                              \
1292          "movq 8(%%rax), %%rdi\n\t"                               \
1293          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1294          VALGRIND_CALL_NOREDIR_RAX                                \
1295          "addq $16, %%rsp\n"                                      \
1296          "addq $128,%%rsp\n\t"                                    \
1297          : /*out*/   "=a" (_res)                                  \
1298          : /*in*/    "a" (&_argvec[0])                            \
1299          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1300       );                                                          \
1301       lval = (__typeof__(lval)) _res;                             \
1302    } while (0)
1303
1304 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1305                                  arg7,arg8,arg9)                  \
1306    do {                                                           \
1307       volatile OrigFn        _orig = (orig);                      \
1308       volatile unsigned long _argvec[10];                         \
1309       volatile unsigned long _res;                                \
1310       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1311       _argvec[1] = (unsigned long)(arg1);                         \
1312       _argvec[2] = (unsigned long)(arg2);                         \
1313       _argvec[3] = (unsigned long)(arg3);                         \
1314       _argvec[4] = (unsigned long)(arg4);                         \
1315       _argvec[5] = (unsigned long)(arg5);                         \
1316       _argvec[6] = (unsigned long)(arg6);                         \
1317       _argvec[7] = (unsigned long)(arg7);                         \
1318       _argvec[8] = (unsigned long)(arg8);                         \
1319       _argvec[9] = (unsigned long)(arg9);                         \
1320       __asm__ volatile(                                           \
1321          "subq $128,%%rsp\n\t"                                    \
1322          "pushq 72(%%rax)\n\t"                                    \
1323          "pushq 64(%%rax)\n\t"                                    \
1324          "pushq 56(%%rax)\n\t"                                    \
1325          "movq 48(%%rax), %%r9\n\t"                               \
1326          "movq 40(%%rax), %%r8\n\t"                               \
1327          "movq 32(%%rax), %%rcx\n\t"                              \
1328          "movq 24(%%rax), %%rdx\n\t"                              \
1329          "movq 16(%%rax), %%rsi\n\t"                              \
1330          "movq 8(%%rax), %%rdi\n\t"                               \
1331          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1332          VALGRIND_CALL_NOREDIR_RAX                                \
1333          "addq $24, %%rsp\n"                                      \
1334          "addq $128,%%rsp\n\t"                                    \
1335          : /*out*/   "=a" (_res)                                  \
1336          : /*in*/    "a" (&_argvec[0])                            \
1337          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1338       );                                                          \
1339       lval = (__typeof__(lval)) _res;                             \
1340    } while (0)
1341
1342 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1343                                   arg7,arg8,arg9,arg10)           \
1344    do {                                                           \
1345       volatile OrigFn        _orig = (orig);                      \
1346       volatile unsigned long _argvec[11];                         \
1347       volatile unsigned long _res;                                \
1348       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1349       _argvec[1] = (unsigned long)(arg1);                         \
1350       _argvec[2] = (unsigned long)(arg2);                         \
1351       _argvec[3] = (unsigned long)(arg3);                         \
1352       _argvec[4] = (unsigned long)(arg4);                         \
1353       _argvec[5] = (unsigned long)(arg5);                         \
1354       _argvec[6] = (unsigned long)(arg6);                         \
1355       _argvec[7] = (unsigned long)(arg7);                         \
1356       _argvec[8] = (unsigned long)(arg8);                         \
1357       _argvec[9] = (unsigned long)(arg9);                         \
1358       _argvec[10] = (unsigned long)(arg10);                       \
1359       __asm__ volatile(                                           \
1360          "subq $128,%%rsp\n\t"                                    \
1361          "pushq 80(%%rax)\n\t"                                    \
1362          "pushq 72(%%rax)\n\t"                                    \
1363          "pushq 64(%%rax)\n\t"                                    \
1364          "pushq 56(%%rax)\n\t"                                    \
1365          "movq 48(%%rax), %%r9\n\t"                               \
1366          "movq 40(%%rax), %%r8\n\t"                               \
1367          "movq 32(%%rax), %%rcx\n\t"                              \
1368          "movq 24(%%rax), %%rdx\n\t"                              \
1369          "movq 16(%%rax), %%rsi\n\t"                              \
1370          "movq 8(%%rax), %%rdi\n\t"                               \
1371          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1372          VALGRIND_CALL_NOREDIR_RAX                                \
1373          "addq $32, %%rsp\n"                                      \
1374          "addq $128,%%rsp\n\t"                                    \
1375          : /*out*/   "=a" (_res)                                  \
1376          : /*in*/    "a" (&_argvec[0])                            \
1377          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1378       );                                                          \
1379       lval = (__typeof__(lval)) _res;                             \
1380    } while (0)
1381
1382 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1383                                   arg7,arg8,arg9,arg10,arg11)     \
1384    do {                                                           \
1385       volatile OrigFn        _orig = (orig);                      \
1386       volatile unsigned long _argvec[12];                         \
1387       volatile unsigned long _res;                                \
1388       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1389       _argvec[1] = (unsigned long)(arg1);                         \
1390       _argvec[2] = (unsigned long)(arg2);                         \
1391       _argvec[3] = (unsigned long)(arg3);                         \
1392       _argvec[4] = (unsigned long)(arg4);                         \
1393       _argvec[5] = (unsigned long)(arg5);                         \
1394       _argvec[6] = (unsigned long)(arg6);                         \
1395       _argvec[7] = (unsigned long)(arg7);                         \
1396       _argvec[8] = (unsigned long)(arg8);                         \
1397       _argvec[9] = (unsigned long)(arg9);                         \
1398       _argvec[10] = (unsigned long)(arg10);                       \
1399       _argvec[11] = (unsigned long)(arg11);                       \
1400       __asm__ volatile(                                           \
1401          "subq $128,%%rsp\n\t"                                    \
1402          "pushq 88(%%rax)\n\t"                                    \
1403          "pushq 80(%%rax)\n\t"                                    \
1404          "pushq 72(%%rax)\n\t"                                    \
1405          "pushq 64(%%rax)\n\t"                                    \
1406          "pushq 56(%%rax)\n\t"                                    \
1407          "movq 48(%%rax), %%r9\n\t"                               \
1408          "movq 40(%%rax), %%r8\n\t"                               \
1409          "movq 32(%%rax), %%rcx\n\t"                              \
1410          "movq 24(%%rax), %%rdx\n\t"                              \
1411          "movq 16(%%rax), %%rsi\n\t"                              \
1412          "movq 8(%%rax), %%rdi\n\t"                               \
1413          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1414          VALGRIND_CALL_NOREDIR_RAX                                \
1415          "addq $40, %%rsp\n"                                      \
1416          "addq $128,%%rsp\n\t"                                    \
1417          : /*out*/   "=a" (_res)                                  \
1418          : /*in*/    "a" (&_argvec[0])                            \
1419          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1420       );                                                          \
1421       lval = (__typeof__(lval)) _res;                             \
1422    } while (0)
1423
1424 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1425                                 arg7,arg8,arg9,arg10,arg11,arg12) \
1426    do {                                                           \
1427       volatile OrigFn        _orig = (orig);                      \
1428       volatile unsigned long _argvec[13];                         \
1429       volatile unsigned long _res;                                \
1430       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1431       _argvec[1] = (unsigned long)(arg1);                         \
1432       _argvec[2] = (unsigned long)(arg2);                         \
1433       _argvec[3] = (unsigned long)(arg3);                         \
1434       _argvec[4] = (unsigned long)(arg4);                         \
1435       _argvec[5] = (unsigned long)(arg5);                         \
1436       _argvec[6] = (unsigned long)(arg6);                         \
1437       _argvec[7] = (unsigned long)(arg7);                         \
1438       _argvec[8] = (unsigned long)(arg8);                         \
1439       _argvec[9] = (unsigned long)(arg9);                         \
1440       _argvec[10] = (unsigned long)(arg10);                       \
1441       _argvec[11] = (unsigned long)(arg11);                       \
1442       _argvec[12] = (unsigned long)(arg12);                       \
1443       __asm__ volatile(                                           \
1444          "subq $128,%%rsp\n\t"                                    \
1445          "pushq 96(%%rax)\n\t"                                    \
1446          "pushq 88(%%rax)\n\t"                                    \
1447          "pushq 80(%%rax)\n\t"                                    \
1448          "pushq 72(%%rax)\n\t"                                    \
1449          "pushq 64(%%rax)\n\t"                                    \
1450          "pushq 56(%%rax)\n\t"                                    \
1451          "movq 48(%%rax), %%r9\n\t"                               \
1452          "movq 40(%%rax), %%r8\n\t"                               \
1453          "movq 32(%%rax), %%rcx\n\t"                              \
1454          "movq 24(%%rax), %%rdx\n\t"                              \
1455          "movq 16(%%rax), %%rsi\n\t"                              \
1456          "movq 8(%%rax), %%rdi\n\t"                               \
1457          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1458          VALGRIND_CALL_NOREDIR_RAX                                \
1459          "addq $48, %%rsp\n"                                      \
1460          "addq $128,%%rsp\n\t"                                    \
1461          : /*out*/   "=a" (_res)                                  \
1462          : /*in*/    "a" (&_argvec[0])                            \
1463          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1464       );                                                          \
1465       lval = (__typeof__(lval)) _res;                             \
1466    } while (0)
1467
1468 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
1469
1470 /* ------------------------ ppc32-linux ------------------------ */
1471
1472 #if defined(PLAT_ppc32_linux)
1473
1474 /* This is useful for finding out about the on-stack stuff:
1475
1476    extern int f9  ( int,int,int,int,int,int,int,int,int );
1477    extern int f10 ( int,int,int,int,int,int,int,int,int,int );
1478    extern int f11 ( int,int,int,int,int,int,int,int,int,int,int );
1479    extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int );
1480
1481    int g9 ( void ) {
1482       return f9(11,22,33,44,55,66,77,88,99);
1483    }
1484    int g10 ( void ) {
1485       return f10(11,22,33,44,55,66,77,88,99,110);
1486    }
1487    int g11 ( void ) {
1488       return f11(11,22,33,44,55,66,77,88,99,110,121);
1489    }
1490    int g12 ( void ) {
1491       return f12(11,22,33,44,55,66,77,88,99,110,121,132);
1492    }
1493 */
1494
1495 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
1496
1497 /* These regs are trashed by the hidden call. */
1498 #define __CALLER_SAVED_REGS                                       \
1499    "lr", "ctr", "xer",                                            \
1500    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
1501    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
1502    "r11", "r12", "r13"
1503
1504 /* These CALL_FN_ macros assume that on ppc32-linux,
1505    sizeof(unsigned long) == 4. */
1506
1507 #define CALL_FN_W_v(lval, orig)                                   \
1508    do {                                                           \
1509       volatile OrigFn        _orig = (orig);                      \
1510       volatile unsigned long _argvec[1];                          \
1511       volatile unsigned long _res;                                \
1512       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1513       __asm__ volatile(                                           \
1514          "mr 11,%1\n\t"                                           \
1515          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1516          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1517          "mr %0,3"                                                \
1518          : /*out*/   "=r" (_res)                                  \
1519          : /*in*/    "r" (&_argvec[0])                            \
1520          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1521       );                                                          \
1522       lval = (__typeof__(lval)) _res;                             \
1523    } while (0)
1524
1525 #define CALL_FN_W_W(lval, orig, arg1)                             \
1526    do {                                                           \
1527       volatile OrigFn        _orig = (orig);                      \
1528       volatile unsigned long _argvec[2];                          \
1529       volatile unsigned long _res;                                \
1530       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1531       _argvec[1] = (unsigned long)arg1;                           \
1532       __asm__ volatile(                                           \
1533          "mr 11,%1\n\t"                                           \
1534          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1535          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1536          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1537          "mr %0,3"                                                \
1538          : /*out*/   "=r" (_res)                                  \
1539          : /*in*/    "r" (&_argvec[0])                            \
1540          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1541       );                                                          \
1542       lval = (__typeof__(lval)) _res;                             \
1543    } while (0)
1544
1545 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1546    do {                                                           \
1547       volatile OrigFn        _orig = (orig);                      \
1548       volatile unsigned long _argvec[3];                          \
1549       volatile unsigned long _res;                                \
1550       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1551       _argvec[1] = (unsigned long)arg1;                           \
1552       _argvec[2] = (unsigned long)arg2;                           \
1553       __asm__ volatile(                                           \
1554          "mr 11,%1\n\t"                                           \
1555          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1556          "lwz 4,8(11)\n\t"                                        \
1557          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1558          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1559          "mr %0,3"                                                \
1560          : /*out*/   "=r" (_res)                                  \
1561          : /*in*/    "r" (&_argvec[0])                            \
1562          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1563       );                                                          \
1564       lval = (__typeof__(lval)) _res;                             \
1565    } while (0)
1566
1567 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1568    do {                                                           \
1569       volatile OrigFn        _orig = (orig);                      \
1570       volatile unsigned long _argvec[4];                          \
1571       volatile unsigned long _res;                                \
1572       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1573       _argvec[1] = (unsigned long)arg1;                           \
1574       _argvec[2] = (unsigned long)arg2;                           \
1575       _argvec[3] = (unsigned long)arg3;                           \
1576       __asm__ volatile(                                           \
1577          "mr 11,%1\n\t"                                           \
1578          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1579          "lwz 4,8(11)\n\t"                                        \
1580          "lwz 5,12(11)\n\t"                                       \
1581          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1582          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1583          "mr %0,3"                                                \
1584          : /*out*/   "=r" (_res)                                  \
1585          : /*in*/    "r" (&_argvec[0])                            \
1586          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1587       );                                                          \
1588       lval = (__typeof__(lval)) _res;                             \
1589    } while (0)
1590
1591 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1592    do {                                                           \
1593       volatile OrigFn        _orig = (orig);                      \
1594       volatile unsigned long _argvec[5];                          \
1595       volatile unsigned long _res;                                \
1596       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1597       _argvec[1] = (unsigned long)arg1;                           \
1598       _argvec[2] = (unsigned long)arg2;                           \
1599       _argvec[3] = (unsigned long)arg3;                           \
1600       _argvec[4] = (unsigned long)arg4;                           \
1601       __asm__ volatile(                                           \
1602          "mr 11,%1\n\t"                                           \
1603          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1604          "lwz 4,8(11)\n\t"                                        \
1605          "lwz 5,12(11)\n\t"                                       \
1606          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1607          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1608          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1609          "mr %0,3"                                                \
1610          : /*out*/   "=r" (_res)                                  \
1611          : /*in*/    "r" (&_argvec[0])                            \
1612          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1613       );                                                          \
1614       lval = (__typeof__(lval)) _res;                             \
1615    } while (0)
1616
1617 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1618    do {                                                           \
1619       volatile OrigFn        _orig = (orig);                      \
1620       volatile unsigned long _argvec[6];                          \
1621       volatile unsigned long _res;                                \
1622       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1623       _argvec[1] = (unsigned long)arg1;                           \
1624       _argvec[2] = (unsigned long)arg2;                           \
1625       _argvec[3] = (unsigned long)arg3;                           \
1626       _argvec[4] = (unsigned long)arg4;                           \
1627       _argvec[5] = (unsigned long)arg5;                           \
1628       __asm__ volatile(                                           \
1629          "mr 11,%1\n\t"                                           \
1630          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1631          "lwz 4,8(11)\n\t"                                        \
1632          "lwz 5,12(11)\n\t"                                       \
1633          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1634          "lwz 7,20(11)\n\t"                                       \
1635          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1636          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1637          "mr %0,3"                                                \
1638          : /*out*/   "=r" (_res)                                  \
1639          : /*in*/    "r" (&_argvec[0])                            \
1640          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1641       );                                                          \
1642       lval = (__typeof__(lval)) _res;                             \
1643    } while (0)
1644
1645 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1646    do {                                                           \
1647       volatile OrigFn        _orig = (orig);                      \
1648       volatile unsigned long _argvec[7];                          \
1649       volatile unsigned long _res;                                \
1650       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1651       _argvec[1] = (unsigned long)arg1;                           \
1652       _argvec[2] = (unsigned long)arg2;                           \
1653       _argvec[3] = (unsigned long)arg3;                           \
1654       _argvec[4] = (unsigned long)arg4;                           \
1655       _argvec[5] = (unsigned long)arg5;                           \
1656       _argvec[6] = (unsigned long)arg6;                           \
1657       __asm__ volatile(                                           \
1658          "mr 11,%1\n\t"                                           \
1659          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1660          "lwz 4,8(11)\n\t"                                        \
1661          "lwz 5,12(11)\n\t"                                       \
1662          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1663          "lwz 7,20(11)\n\t"                                       \
1664          "lwz 8,24(11)\n\t"                                       \
1665          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1666          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1667          "mr %0,3"                                                \
1668          : /*out*/   "=r" (_res)                                  \
1669          : /*in*/    "r" (&_argvec[0])                            \
1670          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1671       );                                                          \
1672       lval = (__typeof__(lval)) _res;                             \
1673    } while (0)
1674
1675 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1676                                  arg7)                            \
1677    do {                                                           \
1678       volatile OrigFn        _orig = (orig);                      \
1679       volatile unsigned long _argvec[8];                          \
1680       volatile unsigned long _res;                                \
1681       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1682       _argvec[1] = (unsigned long)arg1;                           \
1683       _argvec[2] = (unsigned long)arg2;                           \
1684       _argvec[3] = (unsigned long)arg3;                           \
1685       _argvec[4] = (unsigned long)arg4;                           \
1686       _argvec[5] = (unsigned long)arg5;                           \
1687       _argvec[6] = (unsigned long)arg6;                           \
1688       _argvec[7] = (unsigned long)arg7;                           \
1689       __asm__ volatile(                                           \
1690          "mr 11,%1\n\t"                                           \
1691          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1692          "lwz 4,8(11)\n\t"                                        \
1693          "lwz 5,12(11)\n\t"                                       \
1694          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1695          "lwz 7,20(11)\n\t"                                       \
1696          "lwz 8,24(11)\n\t"                                       \
1697          "lwz 9,28(11)\n\t"                                       \
1698          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1699          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1700          "mr %0,3"                                                \
1701          : /*out*/   "=r" (_res)                                  \
1702          : /*in*/    "r" (&_argvec[0])                            \
1703          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1704       );                                                          \
1705       lval = (__typeof__(lval)) _res;                             \
1706    } while (0)
1707
1708 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1709                                  arg7,arg8)                       \
1710    do {                                                           \
1711       volatile OrigFn        _orig = (orig);                      \
1712       volatile unsigned long _argvec[9];                          \
1713       volatile unsigned long _res;                                \
1714       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1715       _argvec[1] = (unsigned long)arg1;                           \
1716       _argvec[2] = (unsigned long)arg2;                           \
1717       _argvec[3] = (unsigned long)arg3;                           \
1718       _argvec[4] = (unsigned long)arg4;                           \
1719       _argvec[5] = (unsigned long)arg5;                           \
1720       _argvec[6] = (unsigned long)arg6;                           \
1721       _argvec[7] = (unsigned long)arg7;                           \
1722       _argvec[8] = (unsigned long)arg8;                           \
1723       __asm__ volatile(                                           \
1724          "mr 11,%1\n\t"                                           \
1725          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1726          "lwz 4,8(11)\n\t"                                        \
1727          "lwz 5,12(11)\n\t"                                       \
1728          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1729          "lwz 7,20(11)\n\t"                                       \
1730          "lwz 8,24(11)\n\t"                                       \
1731          "lwz 9,28(11)\n\t"                                       \
1732          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1733          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1734          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1735          "mr %0,3"                                                \
1736          : /*out*/   "=r" (_res)                                  \
1737          : /*in*/    "r" (&_argvec[0])                            \
1738          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1739       );                                                          \
1740       lval = (__typeof__(lval)) _res;                             \
1741    } while (0)
1742
1743 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1744                                  arg7,arg8,arg9)                  \
1745    do {                                                           \
1746       volatile OrigFn        _orig = (orig);                      \
1747       volatile unsigned long _argvec[10];                         \
1748       volatile unsigned long _res;                                \
1749       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1750       _argvec[1] = (unsigned long)arg1;                           \
1751       _argvec[2] = (unsigned long)arg2;                           \
1752       _argvec[3] = (unsigned long)arg3;                           \
1753       _argvec[4] = (unsigned long)arg4;                           \
1754       _argvec[5] = (unsigned long)arg5;                           \
1755       _argvec[6] = (unsigned long)arg6;                           \
1756       _argvec[7] = (unsigned long)arg7;                           \
1757       _argvec[8] = (unsigned long)arg8;                           \
1758       _argvec[9] = (unsigned long)arg9;                           \
1759       __asm__ volatile(                                           \
1760          "mr 11,%1\n\t"                                           \
1761          "addi 1,1,-16\n\t"                                       \
1762          /* arg9 */                                               \
1763          "lwz 3,36(11)\n\t"                                       \
1764          "stw 3,8(1)\n\t"                                         \
1765          /* args1-8 */                                            \
1766          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1767          "lwz 4,8(11)\n\t"                                        \
1768          "lwz 5,12(11)\n\t"                                       \
1769          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1770          "lwz 7,20(11)\n\t"                                       \
1771          "lwz 8,24(11)\n\t"                                       \
1772          "lwz 9,28(11)\n\t"                                       \
1773          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1774          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1775          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1776          "addi 1,1,16\n\t"                                        \
1777          "mr %0,3"                                                \
1778          : /*out*/   "=r" (_res)                                  \
1779          : /*in*/    "r" (&_argvec[0])                            \
1780          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1781       );                                                          \
1782       lval = (__typeof__(lval)) _res;                             \
1783    } while (0)
1784
1785 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1786                                   arg7,arg8,arg9,arg10)           \
1787    do {                                                           \
1788       volatile OrigFn        _orig = (orig);                      \
1789       volatile unsigned long _argvec[11];                         \
1790       volatile unsigned long _res;                                \
1791       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1792       _argvec[1] = (unsigned long)arg1;                           \
1793       _argvec[2] = (unsigned long)arg2;                           \
1794       _argvec[3] = (unsigned long)arg3;                           \
1795       _argvec[4] = (unsigned long)arg4;                           \
1796       _argvec[5] = (unsigned long)arg5;                           \
1797       _argvec[6] = (unsigned long)arg6;                           \
1798       _argvec[7] = (unsigned long)arg7;                           \
1799       _argvec[8] = (unsigned long)arg8;                           \
1800       _argvec[9] = (unsigned long)arg9;                           \
1801       _argvec[10] = (unsigned long)arg10;                         \
1802       __asm__ volatile(                                           \
1803          "mr 11,%1\n\t"                                           \
1804          "addi 1,1,-16\n\t"                                       \
1805          /* arg10 */                                              \
1806          "lwz 3,40(11)\n\t"                                       \
1807          "stw 3,12(1)\n\t"                                        \
1808          /* arg9 */                                               \
1809          "lwz 3,36(11)\n\t"                                       \
1810          "stw 3,8(1)\n\t"                                         \
1811          /* args1-8 */                                            \
1812          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1813          "lwz 4,8(11)\n\t"                                        \
1814          "lwz 5,12(11)\n\t"                                       \
1815          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1816          "lwz 7,20(11)\n\t"                                       \
1817          "lwz 8,24(11)\n\t"                                       \
1818          "lwz 9,28(11)\n\t"                                       \
1819          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1820          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1821          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1822          "addi 1,1,16\n\t"                                        \
1823          "mr %0,3"                                                \
1824          : /*out*/   "=r" (_res)                                  \
1825          : /*in*/    "r" (&_argvec[0])                            \
1826          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1827       );                                                          \
1828       lval = (__typeof__(lval)) _res;                             \
1829    } while (0)
1830
1831 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1832                                   arg7,arg8,arg9,arg10,arg11)     \
1833    do {                                                           \
1834       volatile OrigFn        _orig = (orig);                      \
1835       volatile unsigned long _argvec[12];                         \
1836       volatile unsigned long _res;                                \
1837       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1838       _argvec[1] = (unsigned long)arg1;                           \
1839       _argvec[2] = (unsigned long)arg2;                           \
1840       _argvec[3] = (unsigned long)arg3;                           \
1841       _argvec[4] = (unsigned long)arg4;                           \
1842       _argvec[5] = (unsigned long)arg5;                           \
1843       _argvec[6] = (unsigned long)arg6;                           \
1844       _argvec[7] = (unsigned long)arg7;                           \
1845       _argvec[8] = (unsigned long)arg8;                           \
1846       _argvec[9] = (unsigned long)arg9;                           \
1847       _argvec[10] = (unsigned long)arg10;                         \
1848       _argvec[11] = (unsigned long)arg11;                         \
1849       __asm__ volatile(                                           \
1850          "mr 11,%1\n\t"                                           \
1851          "addi 1,1,-32\n\t"                                       \
1852          /* arg11 */                                              \
1853          "lwz 3,44(11)\n\t"                                       \
1854          "stw 3,16(1)\n\t"                                        \
1855          /* arg10 */                                              \
1856          "lwz 3,40(11)\n\t"                                       \
1857          "stw 3,12(1)\n\t"                                        \
1858          /* arg9 */                                               \
1859          "lwz 3,36(11)\n\t"                                       \
1860          "stw 3,8(1)\n\t"                                         \
1861          /* args1-8 */                                            \
1862          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1863          "lwz 4,8(11)\n\t"                                        \
1864          "lwz 5,12(11)\n\t"                                       \
1865          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1866          "lwz 7,20(11)\n\t"                                       \
1867          "lwz 8,24(11)\n\t"                                       \
1868          "lwz 9,28(11)\n\t"                                       \
1869          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1870          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1871          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1872          "addi 1,1,32\n\t"                                        \
1873          "mr %0,3"                                                \
1874          : /*out*/   "=r" (_res)                                  \
1875          : /*in*/    "r" (&_argvec[0])                            \
1876          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1877       );                                                          \
1878       lval = (__typeof__(lval)) _res;                             \
1879    } while (0)
1880
1881 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1882                                 arg7,arg8,arg9,arg10,arg11,arg12) \
1883    do {                                                           \
1884       volatile OrigFn        _orig = (orig);                      \
1885       volatile unsigned long _argvec[13];                         \
1886       volatile unsigned long _res;                                \
1887       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1888       _argvec[1] = (unsigned long)arg1;                           \
1889       _argvec[2] = (unsigned long)arg2;                           \
1890       _argvec[3] = (unsigned long)arg3;                           \
1891       _argvec[4] = (unsigned long)arg4;                           \
1892       _argvec[5] = (unsigned long)arg5;                           \
1893       _argvec[6] = (unsigned long)arg6;                           \
1894       _argvec[7] = (unsigned long)arg7;                           \
1895       _argvec[8] = (unsigned long)arg8;                           \
1896       _argvec[9] = (unsigned long)arg9;                           \
1897       _argvec[10] = (unsigned long)arg10;                         \
1898       _argvec[11] = (unsigned long)arg11;                         \
1899       _argvec[12] = (unsigned long)arg12;                         \
1900       __asm__ volatile(                                           \
1901          "mr 11,%1\n\t"                                           \
1902          "addi 1,1,-32\n\t"                                       \
1903          /* arg12 */                                              \
1904          "lwz 3,48(11)\n\t"                                       \
1905          "stw 3,20(1)\n\t"                                        \
1906          /* arg11 */                                              \
1907          "lwz 3,44(11)\n\t"                                       \
1908          "stw 3,16(1)\n\t"                                        \
1909          /* arg10 */                                              \
1910          "lwz 3,40(11)\n\t"                                       \
1911          "stw 3,12(1)\n\t"                                        \
1912          /* arg9 */                                               \
1913          "lwz 3,36(11)\n\t"                                       \
1914          "stw 3,8(1)\n\t"                                         \
1915          /* args1-8 */                                            \
1916          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1917          "lwz 4,8(11)\n\t"                                        \
1918          "lwz 5,12(11)\n\t"                                       \
1919          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1920          "lwz 7,20(11)\n\t"                                       \
1921          "lwz 8,24(11)\n\t"                                       \
1922          "lwz 9,28(11)\n\t"                                       \
1923          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1924          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1925          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1926          "addi 1,1,32\n\t"                                        \
1927          "mr %0,3"                                                \
1928          : /*out*/   "=r" (_res)                                  \
1929          : /*in*/    "r" (&_argvec[0])                            \
1930          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1931       );                                                          \
1932       lval = (__typeof__(lval)) _res;                             \
1933    } while (0)
1934
1935 #endif /* PLAT_ppc32_linux */
1936
1937 /* ------------------------ ppc64-linux ------------------------ */
1938
1939 #if defined(PLAT_ppc64_linux)
1940
1941 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
1942
1943 /* These regs are trashed by the hidden call. */
1944 #define __CALLER_SAVED_REGS                                       \
1945    "lr", "ctr", "xer",                                            \
1946    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
1947    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
1948    "r11", "r12", "r13"
1949
1950 /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
1951    long) == 8. */
1952
1953 #define CALL_FN_W_v(lval, orig)                                   \
1954    do {                                                           \
1955       volatile OrigFn        _orig = (orig);                      \
1956       volatile unsigned long _argvec[3+0];                        \
1957       volatile unsigned long _res;                                \
1958       /* _argvec[0] holds current r2 across the call */           \
1959       _argvec[1] = (unsigned long)_orig.r2;                       \
1960       _argvec[2] = (unsigned long)_orig.nraddr;                   \
1961       __asm__ volatile(                                           \
1962          "mr 11,%1\n\t"                                           \
1963          "std 2,-16(11)\n\t"  /* save tocptr */                   \
1964          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
1965          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
1966          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1967          "mr 11,%1\n\t"                                           \
1968          "mr %0,3\n\t"                                            \
1969          "ld 2,-16(11)" /* restore tocptr */                      \
1970          : /*out*/   "=r" (_res)                                  \
1971          : /*in*/    "r" (&_argvec[2])                            \
1972          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1973       );                                                          \
1974       lval = (__typeof__(lval)) _res;                             \
1975    } while (0)
1976
1977 #define CALL_FN_W_W(lval, orig, arg1)                             \
1978    do {                                                           \
1979       volatile OrigFn        _orig = (orig);                      \
1980       volatile unsigned long _argvec[3+1];                        \
1981       volatile unsigned long _res;                                \
1982       /* _argvec[0] holds current r2 across the call */           \
1983       _argvec[1]   = (unsigned long)_orig.r2;                     \
1984       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
1985       _argvec[2+1] = (unsigned long)arg1;                         \
1986       __asm__ volatile(                                           \
1987          "mr 11,%1\n\t"                                           \
1988          "std 2,-16(11)\n\t"  /* save tocptr */                   \
1989          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
1990          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
1991          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
1992          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1993          "mr 11,%1\n\t"                                           \
1994          "mr %0,3\n\t"                                            \
1995          "ld 2,-16(11)" /* restore tocptr */                      \
1996          : /*out*/   "=r" (_res)                                  \
1997          : /*in*/    "r" (&_argvec[2])                            \
1998          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1999       );                                                          \
2000       lval = (__typeof__(lval)) _res;                             \
2001    } while (0)
2002
2003 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2004    do {                                                           \
2005       volatile OrigFn        _orig = (orig);                      \
2006       volatile unsigned long _argvec[3+2];                        \
2007       volatile unsigned long _res;                                \
2008       /* _argvec[0] holds current r2 across the call */           \
2009       _argvec[1]   = (unsigned long)_orig.r2;                     \
2010       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2011       _argvec[2+1] = (unsigned long)arg1;                         \
2012       _argvec[2+2] = (unsigned long)arg2;                         \
2013       __asm__ volatile(                                           \
2014          "mr 11,%1\n\t"                                           \
2015          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2016          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2017          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2018          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2019          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2020          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2021          "mr 11,%1\n\t"                                           \
2022          "mr %0,3\n\t"                                            \
2023          "ld 2,-16(11)" /* restore tocptr */                      \
2024          : /*out*/   "=r" (_res)                                  \
2025          : /*in*/    "r" (&_argvec[2])                            \
2026          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2027       );                                                          \
2028       lval = (__typeof__(lval)) _res;                             \
2029    } while (0)
2030
2031 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2032    do {                                                           \
2033       volatile OrigFn        _orig = (orig);                      \
2034       volatile unsigned long _argvec[3+3];                        \
2035       volatile unsigned long _res;                                \
2036       /* _argvec[0] holds current r2 across the call */           \
2037       _argvec[1]   = (unsigned long)_orig.r2;                     \
2038       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2039       _argvec[2+1] = (unsigned long)arg1;                         \
2040       _argvec[2+2] = (unsigned long)arg2;                         \
2041       _argvec[2+3] = (unsigned long)arg3;                         \
2042       __asm__ volatile(                                           \
2043          "mr 11,%1\n\t"                                           \
2044          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2045          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2046          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2047          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2048          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2049          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2050          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2051          "mr 11,%1\n\t"                                           \
2052          "mr %0,3\n\t"                                            \
2053          "ld 2,-16(11)" /* restore tocptr */                      \
2054          : /*out*/   "=r" (_res)                                  \
2055          : /*in*/    "r" (&_argvec[2])                            \
2056          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2057       );                                                          \
2058       lval = (__typeof__(lval)) _res;                             \
2059    } while (0)
2060
2061 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2062    do {                                                           \
2063       volatile OrigFn        _orig = (orig);                      \
2064       volatile unsigned long _argvec[3+4];                        \
2065       volatile unsigned long _res;                                \
2066       /* _argvec[0] holds current r2 across the call */           \
2067       _argvec[1]   = (unsigned long)_orig.r2;                     \
2068       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2069       _argvec[2+1] = (unsigned long)arg1;                         \
2070       _argvec[2+2] = (unsigned long)arg2;                         \
2071       _argvec[2+3] = (unsigned long)arg3;                         \
2072       _argvec[2+4] = (unsigned long)arg4;                         \
2073       __asm__ volatile(                                           \
2074          "mr 11,%1\n\t"                                           \
2075          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2076          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2077          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2078          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2079          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2080          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2081          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2082          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2083          "mr 11,%1\n\t"                                           \
2084          "mr %0,3\n\t"                                            \
2085          "ld 2,-16(11)" /* restore tocptr */                      \
2086          : /*out*/   "=r" (_res)                                  \
2087          : /*in*/    "r" (&_argvec[2])                            \
2088          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2089       );                                                          \
2090       lval = (__typeof__(lval)) _res;                             \
2091    } while (0)
2092
2093 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2094    do {                                                           \
2095       volatile OrigFn        _orig = (orig);                      \
2096       volatile unsigned long _argvec[3+5];                        \
2097       volatile unsigned long _res;                                \
2098       /* _argvec[0] holds current r2 across the call */           \
2099       _argvec[1]   = (unsigned long)_orig.r2;                     \
2100       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2101       _argvec[2+1] = (unsigned long)arg1;                         \
2102       _argvec[2+2] = (unsigned long)arg2;                         \
2103       _argvec[2+3] = (unsigned long)arg3;                         \
2104       _argvec[2+4] = (unsigned long)arg4;                         \
2105       _argvec[2+5] = (unsigned long)arg5;                         \
2106       __asm__ volatile(                                           \
2107          "mr 11,%1\n\t"                                           \
2108          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2109          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2110          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2111          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2112          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2113          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2114          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2115          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2116          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2117          "mr 11,%1\n\t"                                           \
2118          "mr %0,3\n\t"                                            \
2119          "ld 2,-16(11)" /* restore tocptr */                      \
2120          : /*out*/   "=r" (_res)                                  \
2121          : /*in*/    "r" (&_argvec[2])                            \
2122          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2123       );                                                          \
2124       lval = (__typeof__(lval)) _res;                             \
2125    } while (0)
2126
2127 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2128    do {                                                           \
2129       volatile OrigFn        _orig = (orig);                      \
2130       volatile unsigned long _argvec[3+6];                        \
2131       volatile unsigned long _res;                                \
2132       /* _argvec[0] holds current r2 across the call */           \
2133       _argvec[1]   = (unsigned long)_orig.r2;                     \
2134       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2135       _argvec[2+1] = (unsigned long)arg1;                         \
2136       _argvec[2+2] = (unsigned long)arg2;                         \
2137       _argvec[2+3] = (unsigned long)arg3;                         \
2138       _argvec[2+4] = (unsigned long)arg4;                         \
2139       _argvec[2+5] = (unsigned long)arg5;                         \
2140       _argvec[2+6] = (unsigned long)arg6;                         \
2141       __asm__ volatile(                                           \
2142          "mr 11,%1\n\t"                                           \
2143          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2144          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2145          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2146          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2147          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2148          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2149          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2150          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2151          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2152          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2153          "mr 11,%1\n\t"                                           \
2154          "mr %0,3\n\t"                                            \
2155          "ld 2,-16(11)" /* restore tocptr */                      \
2156          : /*out*/   "=r" (_res)                                  \
2157          : /*in*/    "r" (&_argvec[2])                            \
2158          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2159       );                                                          \
2160       lval = (__typeof__(lval)) _res;                             \
2161    } while (0)
2162
2163 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2164                                  arg7)                            \
2165    do {                                                           \
2166       volatile OrigFn        _orig = (orig);                      \
2167       volatile unsigned long _argvec[3+7];                        \
2168       volatile unsigned long _res;                                \
2169       /* _argvec[0] holds current r2 across the call */           \
2170       _argvec[1]   = (unsigned long)_orig.r2;                     \
2171       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2172       _argvec[2+1] = (unsigned long)arg1;                         \
2173       _argvec[2+2] = (unsigned long)arg2;                         \
2174       _argvec[2+3] = (unsigned long)arg3;                         \
2175       _argvec[2+4] = (unsigned long)arg4;                         \
2176       _argvec[2+5] = (unsigned long)arg5;                         \
2177       _argvec[2+6] = (unsigned long)arg6;                         \
2178       _argvec[2+7] = (unsigned long)arg7;                         \
2179       __asm__ volatile(                                           \
2180          "mr 11,%1\n\t"                                           \
2181          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2182          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2183          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2184          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2185          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2186          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2187          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2188          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2189          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2190          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2191          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2192          "mr 11,%1\n\t"                                           \
2193          "mr %0,3\n\t"                                            \
2194          "ld 2,-16(11)" /* restore tocptr */                      \
2195          : /*out*/   "=r" (_res)                                  \
2196          : /*in*/    "r" (&_argvec[2])                            \
2197          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2198       );                                                          \
2199       lval = (__typeof__(lval)) _res;                             \
2200    } while (0)
2201
2202 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2203                                  arg7,arg8)                       \
2204    do {                                                           \
2205       volatile OrigFn        _orig = (orig);                      \
2206       volatile unsigned long _argvec[3+8];                        \
2207       volatile unsigned long _res;                                \
2208       /* _argvec[0] holds current r2 across the call */           \
2209       _argvec[1]   = (unsigned long)_orig.r2;                     \
2210       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2211       _argvec[2+1] = (unsigned long)arg1;                         \
2212       _argvec[2+2] = (unsigned long)arg2;                         \
2213       _argvec[2+3] = (unsigned long)arg3;                         \
2214       _argvec[2+4] = (unsigned long)arg4;                         \
2215       _argvec[2+5] = (unsigned long)arg5;                         \
2216       _argvec[2+6] = (unsigned long)arg6;                         \
2217       _argvec[2+7] = (unsigned long)arg7;                         \
2218       _argvec[2+8] = (unsigned long)arg8;                         \
2219       __asm__ volatile(                                           \
2220          "mr 11,%1\n\t"                                           \
2221          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2222          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2223          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2224          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2225          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2226          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2227          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2228          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2229          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2230          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2231          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2232          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2233          "mr 11,%1\n\t"                                           \
2234          "mr %0,3\n\t"                                            \
2235          "ld 2,-16(11)" /* restore tocptr */                      \
2236          : /*out*/   "=r" (_res)                                  \
2237          : /*in*/    "r" (&_argvec[2])                            \
2238          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2239       );                                                          \
2240       lval = (__typeof__(lval)) _res;                             \
2241    } while (0)
2242
2243 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2244                                  arg7,arg8,arg9)                  \
2245    do {                                                           \
2246       volatile OrigFn        _orig = (orig);                      \
2247       volatile unsigned long _argvec[3+9];                        \
2248       volatile unsigned long _res;                                \
2249       /* _argvec[0] holds current r2 across the call */           \
2250       _argvec[1]   = (unsigned long)_orig.r2;                     \
2251       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2252       _argvec[2+1] = (unsigned long)arg1;                         \
2253       _argvec[2+2] = (unsigned long)arg2;                         \
2254       _argvec[2+3] = (unsigned long)arg3;                         \
2255       _argvec[2+4] = (unsigned long)arg4;                         \
2256       _argvec[2+5] = (unsigned long)arg5;                         \
2257       _argvec[2+6] = (unsigned long)arg6;                         \
2258       _argvec[2+7] = (unsigned long)arg7;                         \
2259       _argvec[2+8] = (unsigned long)arg8;                         \
2260       _argvec[2+9] = (unsigned long)arg9;                         \
2261       __asm__ volatile(                                           \
2262          "mr 11,%1\n\t"                                           \
2263          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2264          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2265          "addi 1,1,-128\n\t"  /* expand stack frame */            \
2266          /* arg9 */                                               \
2267          "ld  3,72(11)\n\t"                                       \
2268          "std 3,112(1)\n\t"                                       \
2269          /* args1-8 */                                            \
2270          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2271          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2272          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2273          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2274          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2275          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2276          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2277          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2278          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2279          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2280          "mr 11,%1\n\t"                                           \
2281          "mr %0,3\n\t"                                            \
2282          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2283          "addi 1,1,128"     /* restore frame */                   \
2284          : /*out*/   "=r" (_res)                                  \
2285          : /*in*/    "r" (&_argvec[2])                            \
2286          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2287       );                                                          \
2288       lval = (__typeof__(lval)) _res;                             \
2289    } while (0)
2290
2291 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2292                                   arg7,arg8,arg9,arg10)           \
2293    do {                                                           \
2294       volatile OrigFn        _orig = (orig);                      \
2295       volatile unsigned long _argvec[3+10];                       \
2296       volatile unsigned long _res;                                \
2297       /* _argvec[0] holds current r2 across the call */           \
2298       _argvec[1]   = (unsigned long)_orig.r2;                     \
2299       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2300       _argvec[2+1] = (unsigned long)arg1;                         \
2301       _argvec[2+2] = (unsigned long)arg2;                         \
2302       _argvec[2+3] = (unsigned long)arg3;                         \
2303       _argvec[2+4] = (unsigned long)arg4;                         \
2304       _argvec[2+5] = (unsigned long)arg5;                         \
2305       _argvec[2+6] = (unsigned long)arg6;                         \
2306       _argvec[2+7] = (unsigned long)arg7;                         \
2307       _argvec[2+8] = (unsigned long)arg8;                         \
2308       _argvec[2+9] = (unsigned long)arg9;                         \
2309       _argvec[2+10] = (unsigned long)arg10;                       \
2310       __asm__ volatile(                                           \
2311          "mr 11,%1\n\t"                                           \
2312          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2313          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2314          "addi 1,1,-128\n\t"  /* expand stack frame */            \
2315          /* arg10 */                                              \
2316          "ld  3,80(11)\n\t"                                       \
2317          "std 3,120(1)\n\t"                                       \
2318          /* arg9 */                                               \
2319          "ld  3,72(11)\n\t"                                       \
2320          "std 3,112(1)\n\t"                                       \
2321          /* args1-8 */                                            \
2322          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2323          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2324          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2325          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2326          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2327          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2328          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2329          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2330          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2331          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2332          "mr 11,%1\n\t"                                           \
2333          "mr %0,3\n\t"                                            \
2334          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2335          "addi 1,1,128"     /* restore frame */                   \
2336          : /*out*/   "=r" (_res)                                  \
2337          : /*in*/    "r" (&_argvec[2])                            \
2338          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2339       );                                                          \
2340       lval = (__typeof__(lval)) _res;                             \
2341    } while (0)
2342
2343 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2344                                   arg7,arg8,arg9,arg10,arg11)     \
2345    do {                                                           \
2346       volatile OrigFn        _orig = (orig);                      \
2347       volatile unsigned long _argvec[3+11];                       \
2348       volatile unsigned long _res;                                \
2349       /* _argvec[0] holds current r2 across the call */           \
2350       _argvec[1]   = (unsigned long)_orig.r2;                     \
2351       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2352       _argvec[2+1] = (unsigned long)arg1;                         \
2353       _argvec[2+2] = (unsigned long)arg2;                         \
2354       _argvec[2+3] = (unsigned long)arg3;                         \
2355       _argvec[2+4] = (unsigned long)arg4;                         \
2356       _argvec[2+5] = (unsigned long)arg5;                         \
2357       _argvec[2+6] = (unsigned long)arg6;                         \
2358       _argvec[2+7] = (unsigned long)arg7;                         \
2359       _argvec[2+8] = (unsigned long)arg8;                         \
2360       _argvec[2+9] = (unsigned long)arg9;                         \
2361       _argvec[2+10] = (unsigned long)arg10;                       \
2362       _argvec[2+11] = (unsigned long)arg11;                       \
2363       __asm__ volatile(                                           \
2364          "mr 11,%1\n\t"                                           \
2365          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2366          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2367          "addi 1,1,-144\n\t"  /* expand stack frame */            \
2368          /* arg11 */                                              \
2369          "ld  3,88(11)\n\t"                                       \
2370          "std 3,128(1)\n\t"                                       \
2371          /* arg10 */                                              \
2372          "ld  3,80(11)\n\t"                                       \
2373          "std 3,120(1)\n\t"                                       \
2374          /* arg9 */                                               \
2375          "ld  3,72(11)\n\t"                                       \
2376          "std 3,112(1)\n\t"                                       \
2377          /* args1-8 */                                            \
2378          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2379          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2380          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2381          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2382          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2383          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2384          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2385          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2386          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2387          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2388          "mr 11,%1\n\t"                                           \
2389          "mr %0,3\n\t"                                            \
2390          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2391          "addi 1,1,144"     /* restore frame */                   \
2392          : /*out*/   "=r" (_res)                                  \
2393          : /*in*/    "r" (&_argvec[2])                            \
2394          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2395       );                                                          \
2396       lval = (__typeof__(lval)) _res;                             \
2397    } while (0)
2398
2399 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2400                                 arg7,arg8,arg9,arg10,arg11,arg12) \
2401    do {                                                           \
2402       volatile OrigFn        _orig = (orig);                      \
2403       volatile unsigned long _argvec[3+12];                       \
2404       volatile unsigned long _res;                                \
2405       /* _argvec[0] holds current r2 across the call */           \
2406       _argvec[1]   = (unsigned long)_orig.r2;                     \
2407       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2408       _argvec[2+1] = (unsigned long)arg1;                         \
2409       _argvec[2+2] = (unsigned long)arg2;                         \
2410       _argvec[2+3] = (unsigned long)arg3;                         \
2411       _argvec[2+4] = (unsigned long)arg4;                         \
2412       _argvec[2+5] = (unsigned long)arg5;                         \
2413       _argvec[2+6] = (unsigned long)arg6;                         \
2414       _argvec[2+7] = (unsigned long)arg7;                         \
2415       _argvec[2+8] = (unsigned long)arg8;                         \
2416       _argvec[2+9] = (unsigned long)arg9;                         \
2417       _argvec[2+10] = (unsigned long)arg10;                       \
2418       _argvec[2+11] = (unsigned long)arg11;                       \
2419       _argvec[2+12] = (unsigned long)arg12;                       \
2420       __asm__ volatile(                                           \
2421          "mr 11,%1\n\t"                                           \
2422          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2423          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2424          "addi 1,1,-144\n\t"  /* expand stack frame */            \
2425          /* arg12 */                                              \
2426          "ld  3,96(11)\n\t"                                       \
2427          "std 3,136(1)\n\t"                                       \
2428          /* arg11 */                                              \
2429          "ld  3,88(11)\n\t"                                       \
2430          "std 3,128(1)\n\t"                                       \
2431          /* arg10 */                                              \
2432          "ld  3,80(11)\n\t"                                       \
2433          "std 3,120(1)\n\t"                                       \
2434          /* arg9 */                                               \
2435          "ld  3,72(11)\n\t"                                       \
2436          "std 3,112(1)\n\t"                                       \
2437          /* args1-8 */                                            \
2438          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2439          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2440          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2441          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2442          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2443          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2444          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2445          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2446          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2447          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2448          "mr 11,%1\n\t"                                           \
2449          "mr %0,3\n\t"                                            \
2450          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2451          "addi 1,1,144"     /* restore frame */                   \
2452          : /*out*/   "=r" (_res)                                  \
2453          : /*in*/    "r" (&_argvec[2])                            \
2454          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2455       );                                                          \
2456       lval = (__typeof__(lval)) _res;                             \
2457    } while (0)
2458
2459 #endif /* PLAT_ppc64_linux */
2460
2461 /* ------------------------ ppc32-aix5 ------------------------- */
2462
2463 #if defined(PLAT_ppc32_aix5)
2464
2465 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2466
2467 /* These regs are trashed by the hidden call. */
2468 #define __CALLER_SAVED_REGS                                       \
2469    "lr", "ctr", "xer",                                            \
2470    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
2471    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
2472    "r11", "r12", "r13"
2473
2474 /* Expand the stack frame, copying enough info that unwinding
2475    still works.  Trashes r3. */
2476
2477 #define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr)                      \
2478          "addi 1,1,-" #_n_fr "\n\t"                               \
2479          "lwz  3," #_n_fr "(1)\n\t"                               \
2480          "stw  3,0(1)\n\t"
2481
2482 #define VG_CONTRACT_FRAME_BY(_n_fr)                               \
2483          "addi 1,1," #_n_fr "\n\t"
2484
2485 /* These CALL_FN_ macros assume that on ppc32-aix5, sizeof(unsigned
2486    long) == 4. */
2487
2488 #define CALL_FN_W_v(lval, orig)                                   \
2489    do {                                                           \
2490       volatile OrigFn        _orig = (orig);                      \
2491       volatile unsigned long _argvec[3+0];                        \
2492       volatile unsigned long _res;                                \
2493       /* _argvec[0] holds current r2 across the call */           \
2494       _argvec[1] = (unsigned long)_orig.r2;                       \
2495       _argvec[2] = (unsigned long)_orig.nraddr;                   \
2496       __asm__ volatile(                                           \
2497          "mr 11,%1\n\t"                                           \
2498          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2499          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2500          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2501          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2502          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2503          "mr 11,%1\n\t"                                           \
2504          "mr %0,3\n\t"                                            \
2505          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2506          VG_CONTRACT_FRAME_BY(512)                                \
2507          : /*out*/   "=r" (_res)                                  \
2508          : /*in*/    "r" (&_argvec[2])                            \
2509          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2510       );                                                          \
2511       lval = (__typeof__(lval)) _res;                             \
2512    } while (0)
2513
2514 #define CALL_FN_W_W(lval, orig, arg1)                             \
2515    do {                                                           \
2516       volatile OrigFn        _orig = (orig);                      \
2517       volatile unsigned long _argvec[3+1];                        \
2518       volatile unsigned long _res;                                \
2519       /* _argvec[0] holds current r2 across the call */           \
2520       _argvec[1]   = (unsigned long)_orig.r2;                     \
2521       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2522       _argvec[2+1] = (unsigned long)arg1;                         \
2523       __asm__ volatile(                                           \
2524          "mr 11,%1\n\t"                                           \
2525          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2526          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2527          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2528          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2529          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2530          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2531          "mr 11,%1\n\t"                                           \
2532          "mr %0,3\n\t"                                            \
2533          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2534          VG_CONTRACT_FRAME_BY(512)                                \
2535          : /*out*/   "=r" (_res)                                  \
2536          : /*in*/    "r" (&_argvec[2])                            \
2537          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2538       );                                                          \
2539       lval = (__typeof__(lval)) _res;                             \
2540    } while (0)
2541
2542 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2543    do {                                                           \
2544       volatile OrigFn        _orig = (orig);                      \
2545       volatile unsigned long _argvec[3+2];                        \
2546       volatile unsigned long _res;                                \
2547       /* _argvec[0] holds current r2 across the call */           \
2548       _argvec[1]   = (unsigned long)_orig.r2;                     \
2549       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2550       _argvec[2+1] = (unsigned long)arg1;                         \
2551       _argvec[2+2] = (unsigned long)arg2;                         \
2552       __asm__ volatile(                                           \
2553          "mr 11,%1\n\t"                                           \
2554          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2555          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2556          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2557          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2558          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2559          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2560          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2561          "mr 11,%1\n\t"                                           \
2562          "mr %0,3\n\t"                                            \
2563          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2564          VG_CONTRACT_FRAME_BY(512)                                \
2565          : /*out*/   "=r" (_res)                                  \
2566          : /*in*/    "r" (&_argvec[2])                            \
2567          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2568       );                                                          \
2569       lval = (__typeof__(lval)) _res;                             \
2570    } while (0)
2571
2572 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2573    do {                                                           \
2574       volatile OrigFn        _orig = (orig);                      \
2575       volatile unsigned long _argvec[3+3];                        \
2576       volatile unsigned long _res;                                \
2577       /* _argvec[0] holds current r2 across the call */           \
2578       _argvec[1]   = (unsigned long)_orig.r2;                     \
2579       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2580       _argvec[2+1] = (unsigned long)arg1;                         \
2581       _argvec[2+2] = (unsigned long)arg2;                         \
2582       _argvec[2+3] = (unsigned long)arg3;                         \
2583       __asm__ volatile(                                           \
2584          "mr 11,%1\n\t"                                           \
2585          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2586          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2587          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2588          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2589          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2590          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2591          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2592          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2593          "mr 11,%1\n\t"                                           \
2594          "mr %0,3\n\t"                                            \
2595          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2596          VG_CONTRACT_FRAME_BY(512)                                \
2597          : /*out*/   "=r" (_res)                                  \
2598          : /*in*/    "r" (&_argvec[2])                            \
2599          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2600       );                                                          \
2601       lval = (__typeof__(lval)) _res;                             \
2602    } while (0)
2603
2604 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2605    do {                                                           \
2606       volatile OrigFn        _orig = (orig);                      \
2607       volatile unsigned long _argvec[3+4];                        \
2608       volatile unsigned long _res;                                \
2609       /* _argvec[0] holds current r2 across the call */           \
2610       _argvec[1]   = (unsigned long)_orig.r2;                     \
2611       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2612       _argvec[2+1] = (unsigned long)arg1;                         \
2613       _argvec[2+2] = (unsigned long)arg2;                         \
2614       _argvec[2+3] = (unsigned long)arg3;                         \
2615       _argvec[2+4] = (unsigned long)arg4;                         \
2616       __asm__ volatile(                                           \
2617          "mr 11,%1\n\t"                                           \
2618          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2619          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2620          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2621          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2622          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2623          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2624          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
2625          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2626          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2627          "mr 11,%1\n\t"                                           \
2628          "mr %0,3\n\t"                                            \
2629          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2630          VG_CONTRACT_FRAME_BY(512)                                \
2631          : /*out*/   "=r" (_res)                                  \
2632          : /*in*/    "r" (&_argvec[2])                            \
2633          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2634       );                                                          \
2635       lval = (__typeof__(lval)) _res;                             \
2636    } while (0)
2637
2638 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2639    do {                                                           \
2640       volatile OrigFn        _orig = (orig);                      \
2641       volatile unsigned long _argvec[3+5];                        \
2642       volatile unsigned long _res;                                \
2643       /* _argvec[0] holds current r2 across the call */           \
2644       _argvec[1]   = (unsigned long)_orig.r2;                     \
2645       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2646       _argvec[2+1] = (unsigned long)arg1;                         \
2647       _argvec[2+2] = (unsigned long)arg2;                         \
2648       _argvec[2+3] = (unsigned long)arg3;                         \
2649       _argvec[2+4] = (unsigned long)arg4;                         \
2650       _argvec[2+5] = (unsigned long)arg5;                         \
2651       __asm__ volatile(                                           \
2652          "mr 11,%1\n\t"                                           \
2653          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2654          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2655          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2656          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2657          "lwz  4, 8(11)\n\t" /* arg2->r4 */                       \
2658          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2659          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
2660          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
2661          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2662          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2663          "mr 11,%1\n\t"                                           \
2664          "mr %0,3\n\t"                                            \
2665          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2666          VG_CONTRACT_FRAME_BY(512)                                \
2667          : /*out*/   "=r" (_res)                                  \
2668          : /*in*/    "r" (&_argvec[2])                            \
2669          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2670       );                                                          \
2671       lval = (__typeof__(lval)) _res;                             \
2672    } while (0)
2673
2674 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2675    do {                                                           \
2676       volatile OrigFn        _orig = (orig);                      \
2677       volatile unsigned long _argvec[3+6];                        \
2678       volatile unsigned long _res;                                \
2679       /* _argvec[0] holds current r2 across the call */           \
2680       _argvec[1]   = (unsigned long)_orig.r2;                     \
2681       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2682       _argvec[2+1] = (unsigned long)arg1;                         \
2683       _argvec[2+2] = (unsigned long)arg2;                         \
2684       _argvec[2+3] = (unsigned long)arg3;                         \
2685       _argvec[2+4] = (unsigned long)arg4;                         \
2686       _argvec[2+5] = (unsigned long)arg5;                         \
2687       _argvec[2+6] = (unsigned long)arg6;                         \
2688       __asm__ volatile(                                           \
2689          "mr 11,%1\n\t"                                           \
2690          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2691          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2692          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2693          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2694          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2695          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2696          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
2697          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
2698          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
2699          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2700          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2701          "mr 11,%1\n\t"                                           \
2702          "mr %0,3\n\t"                                            \
2703          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2704          VG_CONTRACT_FRAME_BY(512)                                \
2705          : /*out*/   "=r" (_res)                                  \
2706          : /*in*/    "r" (&_argvec[2])                            \
2707          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2708       );                                                          \
2709       lval = (__typeof__(lval)) _res;                             \
2710    } while (0)
2711
2712 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2713                                  arg7)                            \
2714    do {                                                           \
2715       volatile OrigFn        _orig = (orig);                      \
2716       volatile unsigned long _argvec[3+7];                        \
2717       volatile unsigned long _res;                                \
2718       /* _argvec[0] holds current r2 across the call */           \
2719       _argvec[1]   = (unsigned long)_orig.r2;                     \
2720       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2721       _argvec[2+1] = (unsigned long)arg1;                         \
2722       _argvec[2+2] = (unsigned long)arg2;                         \
2723       _argvec[2+3] = (unsigned long)arg3;                         \
2724       _argvec[2+4] = (unsigned long)arg4;                         \
2725       _argvec[2+5] = (unsigned long)arg5;                         \
2726       _argvec[2+6] = (unsigned long)arg6;                         \
2727       _argvec[2+7] = (unsigned long)arg7;                         \
2728       __asm__ volatile(                                           \
2729          "mr 11,%1\n\t"                                           \
2730          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2731          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2732          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2733          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2734          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2735          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2736          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
2737          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
2738          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
2739          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
2740          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2741          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2742          "mr 11,%1\n\t"                                           \
2743          "mr %0,3\n\t"                                            \
2744          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2745          VG_CONTRACT_FRAME_BY(512)                                \
2746          : /*out*/   "=r" (_res)                                  \
2747          : /*in*/    "r" (&_argvec[2])                            \
2748          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2749       );                                                          \
2750       lval = (__typeof__(lval)) _res;                             \
2751    } while (0)
2752
2753 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2754                                  arg7,arg8)                       \
2755    do {                                                           \
2756       volatile OrigFn        _orig = (orig);                      \
2757       volatile unsigned long _argvec[3+8];                        \
2758       volatile unsigned long _res;                                \
2759       /* _argvec[0] holds current r2 across the call */           \
2760       _argvec[1]   = (unsigned long)_orig.r2;                     \
2761       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2762       _argvec[2+1] = (unsigned long)arg1;                         \
2763       _argvec[2+2] = (unsigned long)arg2;                         \
2764       _argvec[2+3] = (unsigned long)arg3;                         \
2765       _argvec[2+4] = (unsigned long)arg4;                         \
2766       _argvec[2+5] = (unsigned long)arg5;                         \
2767       _argvec[2+6] = (unsigned long)arg6;                         \
2768       _argvec[2+7] = (unsigned long)arg7;                         \
2769       _argvec[2+8] = (unsigned long)arg8;                         \
2770       __asm__ volatile(                                           \
2771          "mr 11,%1\n\t"                                           \
2772          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2773          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2774          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2775          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2776          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2777          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2778          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
2779          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
2780          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
2781          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
2782          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
2783          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2784          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2785          "mr 11,%1\n\t"                                           \
2786          "mr %0,3\n\t"                                            \
2787          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2788          VG_CONTRACT_FRAME_BY(512)                                \
2789          : /*out*/   "=r" (_res)                                  \
2790          : /*in*/    "r" (&_argvec[2])                            \
2791          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2792       );                                                          \
2793       lval = (__typeof__(lval)) _res;                             \
2794    } while (0)
2795
2796 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2797                                  arg7,arg8,arg9)                  \
2798    do {                                                           \
2799       volatile OrigFn        _orig = (orig);                      \
2800       volatile unsigned long _argvec[3+9];                        \
2801       volatile unsigned long _res;                                \
2802       /* _argvec[0] holds current r2 across the call */           \
2803       _argvec[1]   = (unsigned long)_orig.r2;                     \
2804       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2805       _argvec[2+1] = (unsigned long)arg1;                         \
2806       _argvec[2+2] = (unsigned long)arg2;                         \
2807       _argvec[2+3] = (unsigned long)arg3;                         \
2808       _argvec[2+4] = (unsigned long)arg4;                         \
2809       _argvec[2+5] = (unsigned long)arg5;                         \
2810       _argvec[2+6] = (unsigned long)arg6;                         \
2811       _argvec[2+7] = (unsigned long)arg7;                         \
2812       _argvec[2+8] = (unsigned long)arg8;                         \
2813       _argvec[2+9] = (unsigned long)arg9;                         \
2814       __asm__ volatile(                                           \
2815          "mr 11,%1\n\t"                                           \
2816          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2817          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2818          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2819          VG_EXPAND_FRAME_BY_trashes_r3(64)                        \
2820          /* arg9 */                                               \
2821          "lwz 3,36(11)\n\t"                                       \
2822          "stw 3,56(1)\n\t"                                        \
2823          /* args1-8 */                                            \
2824          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2825          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2826          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2827          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
2828          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
2829          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
2830          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
2831          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
2832          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2833          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2834          "mr 11,%1\n\t"                                           \
2835          "mr %0,3\n\t"                                            \
2836          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2837          VG_CONTRACT_FRAME_BY(64)                                 \
2838          VG_CONTRACT_FRAME_BY(512)                                \
2839          : /*out*/   "=r" (_res)                                  \
2840          : /*in*/    "r" (&_argvec[2])                            \
2841          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2842       );                                                          \
2843       lval = (__typeof__(lval)) _res;                             \
2844    } while (0)
2845
2846 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2847                                   arg7,arg8,arg9,arg10)           \
2848    do {                                                           \
2849       volatile OrigFn        _orig = (orig);                      \
2850       volatile unsigned long _argvec[3+10];                       \
2851       volatile unsigned long _res;                                \
2852       /* _argvec[0] holds current r2 across the call */           \
2853       _argvec[1]   = (unsigned long)_orig.r2;                     \
2854       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2855       _argvec[2+1] = (unsigned long)arg1;                         \
2856       _argvec[2+2] = (unsigned long)arg2;                         \
2857       _argvec[2+3] = (unsigned long)arg3;                         \
2858       _argvec[2+4] = (unsigned long)arg4;                         \
2859       _argvec[2+5] = (unsigned long)arg5;                         \
2860       _argvec[2+6] = (unsigned long)arg6;                         \
2861       _argvec[2+7] = (unsigned long)arg7;                         \
2862       _argvec[2+8] = (unsigned long)arg8;                         \
2863       _argvec[2+9] = (unsigned long)arg9;                         \
2864       _argvec[2+10] = (unsigned long)arg10;                       \
2865       __asm__ volatile(                                           \
2866          "mr 11,%1\n\t"                                           \
2867          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2868          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2869          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2870          VG_EXPAND_FRAME_BY_trashes_r3(64)                        \
2871          /* arg10 */                                              \
2872          "lwz 3,40(11)\n\t"                                       \
2873          "stw 3,60(1)\n\t"                                        \
2874          /* arg9 */                                               \
2875          "lwz 3,36(11)\n\t"                                       \
2876          "stw 3,56(1)\n\t"                                        \
2877          /* args1-8 */                                            \
2878          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2879          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2880          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2881          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
2882          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
2883          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
2884          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
2885          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
2886          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2887          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2888          "mr 11,%1\n\t"                                           \
2889          "mr %0,3\n\t"                                            \
2890          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2891          VG_CONTRACT_FRAME_BY(64)                                 \
2892          VG_CONTRACT_FRAME_BY(512)                                \
2893          : /*out*/   "=r" (_res)                                  \
2894          : /*in*/    "r" (&_argvec[2])                            \
2895          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2896       );                                                          \
2897       lval = (__typeof__(lval)) _res;                             \
2898    } while (0)
2899
2900 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2901                                   arg7,arg8,arg9,arg10,arg11)     \
2902    do {                                                           \
2903       volatile OrigFn        _orig = (orig);                      \
2904       volatile unsigned long _argvec[3+11];                       \
2905       volatile unsigned long _res;                                \
2906       /* _argvec[0] holds current r2 across the call */           \
2907       _argvec[1]   = (unsigned long)_orig.r2;                     \
2908       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2909       _argvec[2+1] = (unsigned long)arg1;                         \
2910       _argvec[2+2] = (unsigned long)arg2;                         \
2911       _argvec[2+3] = (unsigned long)arg3;                         \
2912       _argvec[2+4] = (unsigned long)arg4;                         \
2913       _argvec[2+5] = (unsigned long)arg5;                         \
2914       _argvec[2+6] = (unsigned long)arg6;                         \
2915       _argvec[2+7] = (unsigned long)arg7;                         \
2916       _argvec[2+8] = (unsigned long)arg8;                         \
2917       _argvec[2+9] = (unsigned long)arg9;                         \
2918       _argvec[2+10] = (unsigned long)arg10;                       \
2919       _argvec[2+11] = (unsigned long)arg11;                       \
2920       __asm__ volatile(                                           \
2921          "mr 11,%1\n\t"                                           \
2922          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2923          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2924          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2925          VG_EXPAND_FRAME_BY_trashes_r3(72)                        \
2926          /* arg11 */                                              \
2927          "lwz 3,44(11)\n\t"                                       \
2928          "stw 3,64(1)\n\t"                                        \
2929          /* arg10 */                                              \
2930          "lwz 3,40(11)\n\t"                                       \
2931          "stw 3,60(1)\n\t"                                        \
2932          /* arg9 */                                               \
2933          "lwz 3,36(11)\n\t"                                       \
2934          "stw 3,56(1)\n\t"                                        \
2935          /* args1-8 */                                            \
2936          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2937          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2938          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2939          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
2940          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
2941          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
2942          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
2943          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
2944          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2945          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2946          "mr 11,%1\n\t"                                           \
2947          "mr %0,3\n\t"                                            \
2948          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2949          VG_CONTRACT_FRAME_BY(72)                                 \
2950          VG_CONTRACT_FRAME_BY(512)                                \
2951          : /*out*/   "=r" (_res)                                  \
2952          : /*in*/    "r" (&_argvec[2])                            \
2953          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2954       );                                                          \
2955       lval = (__typeof__(lval)) _res;                             \
2956    } while (0)
2957
2958 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2959                                 arg7,arg8,arg9,arg10,arg11,arg12) \
2960    do {                                                           \
2961       volatile OrigFn        _orig = (orig);                      \
2962       volatile unsigned long _argvec[3+12];                       \
2963       volatile unsigned long _res;                                \
2964       /* _argvec[0] holds current r2 across the call */           \
2965       _argvec[1]   = (unsigned long)_orig.r2;                     \
2966       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2967       _argvec[2+1] = (unsigned long)arg1;                         \
2968       _argvec[2+2] = (unsigned long)arg2;                         \
2969       _argvec[2+3] = (unsigned long)arg3;                         \
2970       _argvec[2+4] = (unsigned long)arg4;                         \
2971       _argvec[2+5] = (unsigned long)arg5;                         \
2972       _argvec[2+6] = (unsigned long)arg6;                         \
2973       _argvec[2+7] = (unsigned long)arg7;                         \
2974       _argvec[2+8] = (unsigned long)arg8;                         \
2975       _argvec[2+9] = (unsigned long)arg9;                         \
2976       _argvec[2+10] = (unsigned long)arg10;                       \
2977       _argvec[2+11] = (unsigned long)arg11;                       \
2978       _argvec[2+12] = (unsigned long)arg12;                       \
2979       __asm__ volatile(                                           \
2980          "mr 11,%1\n\t"                                           \
2981          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2982          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2983          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2984          VG_EXPAND_FRAME_BY_trashes_r3(72)                        \
2985          /* arg12 */                                              \
2986          "lwz 3,48(11)\n\t"                                       \
2987          "stw 3,68(1)\n\t"                                        \
2988          /* arg11 */                                              \
2989          "lwz 3,44(11)\n\t"                                       \
2990          "stw 3,64(1)\n\t"                                        \
2991          /* arg10 */                                              \
2992          "lwz 3,40(11)\n\t"                                       \
2993          "stw 3,60(1)\n\t"                                        \
2994          /* arg9 */                                               \
2995          "lwz 3,36(11)\n\t"                                       \
2996          "stw 3,56(1)\n\t"                                        \
2997          /* args1-8 */                                            \
2998          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2999          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3000          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3001          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3002          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3003          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3004          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3005          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3006          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3007          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3008          "mr 11,%1\n\t"                                           \
3009          "mr %0,3\n\t"                                            \
3010          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3011          VG_CONTRACT_FRAME_BY(72)                                 \
3012          VG_CONTRACT_FRAME_BY(512)                                \
3013          : /*out*/   "=r" (_res)                                  \
3014          : /*in*/    "r" (&_argvec[2])                            \
3015          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3016       );                                                          \
3017       lval = (__typeof__(lval)) _res;                             \
3018    } while (0)
3019
3020 #endif /* PLAT_ppc32_aix5 */
3021
3022 /* ------------------------ ppc64-aix5 ------------------------- */
3023
3024 #if defined(PLAT_ppc64_aix5)
3025
3026 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
3027
3028 /* These regs are trashed by the hidden call. */
3029 #define __CALLER_SAVED_REGS                                       \
3030    "lr", "ctr", "xer",                                            \
3031    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
3032    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
3033    "r11", "r12", "r13"
3034
3035 /* Expand the stack frame, copying enough info that unwinding
3036    still works.  Trashes r3. */
3037
3038 #define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr)                      \
3039          "addi 1,1,-" #_n_fr "\n\t"                               \
3040          "ld   3," #_n_fr "(1)\n\t"                               \
3041          "std  3,0(1)\n\t"
3042
3043 #define VG_CONTRACT_FRAME_BY(_n_fr)                               \
3044          "addi 1,1," #_n_fr "\n\t"
3045
3046 /* These CALL_FN_ macros assume that on ppc64-aix5, sizeof(unsigned
3047    long) == 8. */
3048
3049 #define CALL_FN_W_v(lval, orig)                                   \
3050    do {                                                           \
3051       volatile OrigFn        _orig = (orig);                      \
3052       volatile unsigned long _argvec[3+0];                        \
3053       volatile unsigned long _res;                                \
3054       /* _argvec[0] holds current r2 across the call */           \
3055       _argvec[1] = (unsigned long)_orig.r2;                       \
3056       _argvec[2] = (unsigned long)_orig.nraddr;                   \
3057       __asm__ volatile(                                           \
3058          "mr 11,%1\n\t"                                           \
3059          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3060          "std  2,-16(11)\n\t" /* save tocptr */                   \
3061          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3062          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3063          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3064          "mr 11,%1\n\t"                                           \
3065          "mr %0,3\n\t"                                            \
3066          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3067          VG_CONTRACT_FRAME_BY(512)                                \
3068          : /*out*/   "=r" (_res)                                  \
3069          : /*in*/    "r" (&_argvec[2])                            \
3070          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3071       );                                                          \
3072       lval = (__typeof__(lval)) _res;                             \
3073    } while (0)
3074
3075 #define CALL_FN_W_W(lval, orig, arg1)                             \
3076    do {                                                           \
3077       volatile OrigFn        _orig = (orig);                      \
3078       volatile unsigned long _argvec[3+1];                        \
3079       volatile unsigned long _res;                                \
3080       /* _argvec[0] holds current r2 across the call */           \
3081       _argvec[1]   = (unsigned long)_orig.r2;                     \
3082       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3083       _argvec[2+1] = (unsigned long)arg1;                         \
3084       __asm__ volatile(                                           \
3085          "mr 11,%1\n\t"                                           \
3086          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3087          "std  2,-16(11)\n\t" /* save tocptr */                   \
3088          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3089          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3090          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3091          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3092          "mr 11,%1\n\t"                                           \
3093          "mr %0,3\n\t"                                            \
3094          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3095          VG_CONTRACT_FRAME_BY(512)                                \
3096          : /*out*/   "=r" (_res)                                  \
3097          : /*in*/    "r" (&_argvec[2])                            \
3098          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3099       );                                                          \
3100       lval = (__typeof__(lval)) _res;                             \
3101    } while (0)
3102
3103 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3104    do {                                                           \
3105       volatile OrigFn        _orig = (orig);                      \
3106       volatile unsigned long _argvec[3+2];                        \
3107       volatile unsigned long _res;                                \
3108       /* _argvec[0] holds current r2 across the call */           \
3109       _argvec[1]   = (unsigned long)_orig.r2;                     \
3110       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3111       _argvec[2+1] = (unsigned long)arg1;                         \
3112       _argvec[2+2] = (unsigned long)arg2;                         \
3113       __asm__ volatile(                                           \
3114          "mr 11,%1\n\t"                                           \
3115          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3116          "std  2,-16(11)\n\t" /* save tocptr */                   \
3117          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3118          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3119          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3120          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3121          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3122          "mr 11,%1\n\t"                                           \
3123          "mr %0,3\n\t"                                            \
3124          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3125          VG_CONTRACT_FRAME_BY(512)                                \
3126          : /*out*/   "=r" (_res)                                  \
3127          : /*in*/    "r" (&_argvec[2])                            \
3128          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3129       );                                                          \
3130       lval = (__typeof__(lval)) _res;                             \
3131    } while (0)
3132
3133 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3134    do {                                                           \
3135       volatile OrigFn        _orig = (orig);                      \
3136       volatile unsigned long _argvec[3+3];                        \
3137       volatile unsigned long _res;                                \
3138       /* _argvec[0] holds current r2 across the call */           \
3139       _argvec[1]   = (unsigned long)_orig.r2;                     \
3140       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3141       _argvec[2+1] = (unsigned long)arg1;                         \
3142       _argvec[2+2] = (unsigned long)arg2;                         \
3143       _argvec[2+3] = (unsigned long)arg3;                         \
3144       __asm__ volatile(                                           \
3145          "mr 11,%1\n\t"                                           \
3146          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3147          "std  2,-16(11)\n\t" /* save tocptr */                   \
3148          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3149          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3150          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3151          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3152          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3153          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3154          "mr 11,%1\n\t"                                           \
3155          "mr %0,3\n\t"                                            \
3156          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3157          VG_CONTRACT_FRAME_BY(512)                                \
3158          : /*out*/   "=r" (_res)                                  \
3159          : /*in*/    "r" (&_argvec[2])                            \
3160          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3161       );                                                          \
3162       lval = (__typeof__(lval)) _res;                             \
3163    } while (0)
3164
3165 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3166    do {                                                           \
3167       volatile OrigFn        _orig = (orig);                      \
3168       volatile unsigned long _argvec[3+4];                        \
3169       volatile unsigned long _res;                                \
3170       /* _argvec[0] holds current r2 across the call */           \
3171       _argvec[1]   = (unsigned long)_orig.r2;                     \
3172       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3173       _argvec[2+1] = (unsigned long)arg1;                         \
3174       _argvec[2+2] = (unsigned long)arg2;                         \
3175       _argvec[2+3] = (unsigned long)arg3;                         \
3176       _argvec[2+4] = (unsigned long)arg4;                         \
3177       __asm__ volatile(                                           \
3178          "mr 11,%1\n\t"                                           \
3179          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3180          "std  2,-16(11)\n\t" /* save tocptr */                   \
3181          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3182          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3183          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3184          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3185          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3186          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3187          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3188          "mr 11,%1\n\t"                                           \
3189          "mr %0,3\n\t"                                            \
3190          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3191          VG_CONTRACT_FRAME_BY(512)                                \
3192          : /*out*/   "=r" (_res)                                  \
3193          : /*in*/    "r" (&_argvec[2])                            \
3194          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3195       );                                                          \
3196       lval = (__typeof__(lval)) _res;                             \
3197    } while (0)
3198
3199 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3200    do {                                                           \
3201       volatile OrigFn        _orig = (orig);                      \
3202       volatile unsigned long _argvec[3+5];                        \
3203       volatile unsigned long _res;                                \
3204       /* _argvec[0] holds current r2 across the call */           \
3205       _argvec[1]   = (unsigned long)_orig.r2;                     \
3206       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3207       _argvec[2+1] = (unsigned long)arg1;                         \
3208       _argvec[2+2] = (unsigned long)arg2;                         \
3209       _argvec[2+3] = (unsigned long)arg3;                         \
3210       _argvec[2+4] = (unsigned long)arg4;                         \
3211       _argvec[2+5] = (unsigned long)arg5;                         \
3212       __asm__ volatile(                                           \
3213          "mr 11,%1\n\t"                                           \
3214          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3215          "std  2,-16(11)\n\t" /* save tocptr */                   \
3216          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3217          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3218          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3219          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3220          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3221          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3222          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3223          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3224          "mr 11,%1\n\t"                                           \
3225          "mr %0,3\n\t"                                            \
3226          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3227          VG_CONTRACT_FRAME_BY(512)                                \
3228          : /*out*/   "=r" (_res)                                  \
3229          : /*in*/    "r" (&_argvec[2])                            \
3230          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3231       );                                                          \
3232       lval = (__typeof__(lval)) _res;                             \
3233    } while (0)
3234
3235 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3236    do {                                                           \
3237       volatile OrigFn        _orig = (orig);                      \
3238       volatile unsigned long _argvec[3+6];                        \
3239       volatile unsigned long _res;                                \
3240       /* _argvec[0] holds current r2 across the call */           \
3241       _argvec[1]   = (unsigned long)_orig.r2;                     \
3242       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3243       _argvec[2+1] = (unsigned long)arg1;                         \
3244       _argvec[2+2] = (unsigned long)arg2;                         \
3245       _argvec[2+3] = (unsigned long)arg3;                         \
3246       _argvec[2+4] = (unsigned long)arg4;                         \
3247       _argvec[2+5] = (unsigned long)arg5;                         \
3248       _argvec[2+6] = (unsigned long)arg6;                         \
3249       __asm__ volatile(                                           \
3250          "mr 11,%1\n\t"                                           \
3251          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3252          "std  2,-16(11)\n\t" /* save tocptr */                   \
3253          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3254          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3255          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3256          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3257          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3258          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3259          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3260          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3261          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3262          "mr 11,%1\n\t"                                           \
3263          "mr %0,3\n\t"                                            \
3264          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3265          VG_CONTRACT_FRAME_BY(512)                                \
3266          : /*out*/   "=r" (_res)                                  \
3267          : /*in*/    "r" (&_argvec[2])                            \
3268          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3269       );                                                          \
3270       lval = (__typeof__(lval)) _res;                             \
3271    } while (0)
3272
3273 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3274                                  arg7)                            \
3275    do {                                                           \
3276       volatile OrigFn        _orig = (orig);                      \
3277       volatile unsigned long _argvec[3+7];                        \
3278       volatile unsigned long _res;                                \
3279       /* _argvec[0] holds current r2 across the call */           \
3280       _argvec[1]   = (unsigned long)_orig.r2;                     \
3281       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3282       _argvec[2+1] = (unsigned long)arg1;                         \
3283       _argvec[2+2] = (unsigned long)arg2;                         \
3284       _argvec[2+3] = (unsigned long)arg3;                         \
3285       _argvec[2+4] = (unsigned long)arg4;                         \
3286       _argvec[2+5] = (unsigned long)arg5;                         \
3287       _argvec[2+6] = (unsigned long)arg6;                         \
3288       _argvec[2+7] = (unsigned long)arg7;                         \
3289       __asm__ volatile(                                           \
3290          "mr 11,%1\n\t"                                           \
3291          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3292          "std  2,-16(11)\n\t" /* save tocptr */                   \
3293          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3294          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3295          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3296          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3297          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3298          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3299          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3300          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3301          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3302          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3303          "mr 11,%1\n\t"                                           \
3304          "mr %0,3\n\t"                                            \
3305          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3306          VG_CONTRACT_FRAME_BY(512)                                \
3307          : /*out*/   "=r" (_res)                                  \
3308          : /*in*/    "r" (&_argvec[2])                            \
3309          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3310       );                                                          \
3311       lval = (__typeof__(lval)) _res;                             \
3312    } while (0)
3313
3314 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3315                                  arg7,arg8)                       \
3316    do {                                                           \
3317       volatile OrigFn        _orig = (orig);                      \
3318       volatile unsigned long _argvec[3+8];                        \
3319       volatile unsigned long _res;                                \
3320       /* _argvec[0] holds current r2 across the call */           \
3321       _argvec[1]   = (unsigned long)_orig.r2;                     \
3322       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3323       _argvec[2+1] = (unsigned long)arg1;                         \
3324       _argvec[2+2] = (unsigned long)arg2;                         \
3325       _argvec[2+3] = (unsigned long)arg3;                         \
3326       _argvec[2+4] = (unsigned long)arg4;                         \
3327       _argvec[2+5] = (unsigned long)arg5;                         \
3328       _argvec[2+6] = (unsigned long)arg6;                         \
3329       _argvec[2+7] = (unsigned long)arg7;                         \
3330       _argvec[2+8] = (unsigned long)arg8;                         \
3331       __asm__ volatile(                                           \
3332          "mr 11,%1\n\t"                                           \
3333          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3334          "std  2,-16(11)\n\t" /* save tocptr */                   \
3335          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3336          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3337          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3338          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3339          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3340          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3341          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3342          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3343          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3344          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3345          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3346          "mr 11,%1\n\t"                                           \
3347          "mr %0,3\n\t"                                            \
3348          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3349          VG_CONTRACT_FRAME_BY(512)                                \
3350          : /*out*/   "=r" (_res)                                  \
3351          : /*in*/    "r" (&_argvec[2])                            \
3352          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3353       );                                                          \
3354       lval = (__typeof__(lval)) _res;                             \
3355    } while (0)
3356
3357 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3358                                  arg7,arg8,arg9)                  \
3359    do {                                                           \
3360       volatile OrigFn        _orig = (orig);                      \
3361       volatile unsigned long _argvec[3+9];                        \
3362       volatile unsigned long _res;                                \
3363       /* _argvec[0] holds current r2 across the call */           \
3364       _argvec[1]   = (unsigned long)_orig.r2;                     \
3365       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3366       _argvec[2+1] = (unsigned long)arg1;                         \
3367       _argvec[2+2] = (unsigned long)arg2;                         \
3368       _argvec[2+3] = (unsigned long)arg3;                         \
3369       _argvec[2+4] = (unsigned long)arg4;                         \
3370       _argvec[2+5] = (unsigned long)arg5;                         \
3371       _argvec[2+6] = (unsigned long)arg6;                         \
3372       _argvec[2+7] = (unsigned long)arg7;                         \
3373       _argvec[2+8] = (unsigned long)arg8;                         \
3374       _argvec[2+9] = (unsigned long)arg9;                         \
3375       __asm__ volatile(                                           \
3376          "mr 11,%1\n\t"                                           \
3377          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3378          "std  2,-16(11)\n\t" /* save tocptr */                   \
3379          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3380          VG_EXPAND_FRAME_BY_trashes_r3(128)                       \
3381          /* arg9 */                                               \
3382          "ld  3,72(11)\n\t"                                       \
3383          "std 3,112(1)\n\t"                                       \
3384          /* args1-8 */                                            \
3385          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3386          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3387          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3388          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3389          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3390          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3391          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3392          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3393          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3394          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3395          "mr 11,%1\n\t"                                           \
3396          "mr %0,3\n\t"                                            \
3397          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3398          VG_CONTRACT_FRAME_BY(128)                                \
3399          VG_CONTRACT_FRAME_BY(512)                                \
3400          : /*out*/   "=r" (_res)                                  \
3401          : /*in*/    "r" (&_argvec[2])                            \
3402          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3403       );                                                          \
3404       lval = (__typeof__(lval)) _res;                             \
3405    } while (0)
3406
3407 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3408                                   arg7,arg8,arg9,arg10)           \
3409    do {                                                           \
3410       volatile OrigFn        _orig = (orig);                      \
3411       volatile unsigned long _argvec[3+10];                       \
3412       volatile unsigned long _res;                                \
3413       /* _argvec[0] holds current r2 across the call */           \
3414       _argvec[1]   = (unsigned long)_orig.r2;                     \
3415       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3416       _argvec[2+1] = (unsigned long)arg1;                         \
3417       _argvec[2+2] = (unsigned long)arg2;                         \
3418       _argvec[2+3] = (unsigned long)arg3;                         \
3419       _argvec[2+4] = (unsigned long)arg4;                         \
3420       _argvec[2+5] = (unsigned long)arg5;                         \
3421       _argvec[2+6] = (unsigned long)arg6;                         \
3422       _argvec[2+7] = (unsigned long)arg7;                         \
3423       _argvec[2+8] = (unsigned long)arg8;                         \
3424       _argvec[2+9] = (unsigned long)arg9;                         \
3425       _argvec[2+10] = (unsigned long)arg10;                       \
3426       __asm__ volatile(                                           \
3427          "mr 11,%1\n\t"                                           \
3428          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3429          "std  2,-16(11)\n\t" /* save tocptr */                   \
3430          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3431          VG_EXPAND_FRAME_BY_trashes_r3(128)                       \
3432          /* arg10 */                                              \
3433          "ld  3,80(11)\n\t"                                       \
3434          "std 3,120(1)\n\t"                                       \
3435          /* arg9 */                                               \
3436          "ld  3,72(11)\n\t"                                       \
3437          "std 3,112(1)\n\t"                                       \
3438          /* args1-8 */                                            \
3439          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3440          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3441          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3442          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3443          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3444          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3445          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3446          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3447          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3448          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3449          "mr 11,%1\n\t"                                           \
3450          "mr %0,3\n\t"                                            \
3451          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3452          VG_CONTRACT_FRAME_BY(128)                                \
3453          VG_CONTRACT_FRAME_BY(512)                                \
3454          : /*out*/   "=r" (_res)                                  \
3455          : /*in*/    "r" (&_argvec[2])                            \
3456          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3457       );                                                          \
3458       lval = (__typeof__(lval)) _res;                             \
3459    } while (0)
3460
3461 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3462                                   arg7,arg8,arg9,arg10,arg11)     \
3463    do {                                                           \
3464       volatile OrigFn        _orig = (orig);                      \
3465       volatile unsigned long _argvec[3+11];                       \
3466       volatile unsigned long _res;                                \
3467       /* _argvec[0] holds current r2 across the call */           \
3468       _argvec[1]   = (unsigned long)_orig.r2;                     \
3469       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3470       _argvec[2+1] = (unsigned long)arg1;                         \
3471       _argvec[2+2] = (unsigned long)arg2;                         \
3472       _argvec[2+3] = (unsigned long)arg3;                         \
3473       _argvec[2+4] = (unsigned long)arg4;                         \
3474       _argvec[2+5] = (unsigned long)arg5;                         \
3475       _argvec[2+6] = (unsigned long)arg6;                         \
3476       _argvec[2+7] = (unsigned long)arg7;                         \
3477       _argvec[2+8] = (unsigned long)arg8;                         \
3478       _argvec[2+9] = (unsigned long)arg9;                         \
3479       _argvec[2+10] = (unsigned long)arg10;                       \
3480       _argvec[2+11] = (unsigned long)arg11;                       \
3481       __asm__ volatile(                                           \
3482          "mr 11,%1\n\t"                                           \
3483          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3484          "std  2,-16(11)\n\t" /* save tocptr */                   \
3485          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3486          VG_EXPAND_FRAME_BY_trashes_r3(144)                       \
3487          /* arg11 */                                              \
3488          "ld  3,88(11)\n\t"                                       \
3489          "std 3,128(1)\n\t"                                       \
3490          /* arg10 */                                              \
3491          "ld  3,80(11)\n\t"                                       \
3492          "std 3,120(1)\n\t"                                       \
3493          /* arg9 */                                               \
3494          "ld  3,72(11)\n\t"                                       \
3495          "std 3,112(1)\n\t"                                       \
3496          /* args1-8 */                                            \
3497          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3498          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3499          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3500          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3501          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3502          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3503          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3504          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3505          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3506          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3507          "mr 11,%1\n\t"                                           \
3508          "mr %0,3\n\t"                                            \
3509          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3510          VG_CONTRACT_FRAME_BY(144)                                \
3511          VG_CONTRACT_FRAME_BY(512)                                \
3512          : /*out*/   "=r" (_res)                                  \
3513          : /*in*/    "r" (&_argvec[2])                            \
3514          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3515       );                                                          \
3516       lval = (__typeof__(lval)) _res;                             \
3517    } while (0)
3518
3519 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3520                                 arg7,arg8,arg9,arg10,arg11,arg12) \
3521    do {                                                           \
3522       volatile OrigFn        _orig = (orig);                      \
3523       volatile unsigned long _argvec[3+12];                       \
3524       volatile unsigned long _res;                                \
3525       /* _argvec[0] holds current r2 across the call */           \
3526       _argvec[1]   = (unsigned long)_orig.r2;                     \
3527       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3528       _argvec[2+1] = (unsigned long)arg1;                         \
3529       _argvec[2+2] = (unsigned long)arg2;                         \
3530       _argvec[2+3] = (unsigned long)arg3;                         \
3531       _argvec[2+4] = (unsigned long)arg4;                         \
3532       _argvec[2+5] = (unsigned long)arg5;                         \
3533       _argvec[2+6] = (unsigned long)arg6;                         \
3534       _argvec[2+7] = (unsigned long)arg7;                         \
3535       _argvec[2+8] = (unsigned long)arg8;                         \
3536       _argvec[2+9] = (unsigned long)arg9;                         \
3537       _argvec[2+10] = (unsigned long)arg10;                       \
3538       _argvec[2+11] = (unsigned long)arg11;                       \
3539       _argvec[2+12] = (unsigned long)arg12;                       \
3540       __asm__ volatile(                                           \
3541          "mr 11,%1\n\t"                                           \
3542          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3543          "std  2,-16(11)\n\t" /* save tocptr */                   \
3544          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3545          VG_EXPAND_FRAME_BY_trashes_r3(144)                       \
3546          /* arg12 */                                              \
3547          "ld  3,96(11)\n\t"                                       \
3548          "std 3,136(1)\n\t"                                       \
3549          /* arg11 */                                              \
3550          "ld  3,88(11)\n\t"                                       \
3551          "std 3,128(1)\n\t"                                       \
3552          /* arg10 */                                              \
3553          "ld  3,80(11)\n\t"                                       \
3554          "std 3,120(1)\n\t"                                       \
3555          /* arg9 */                                               \
3556          "ld  3,72(11)\n\t"                                       \
3557          "std 3,112(1)\n\t"                                       \
3558          /* args1-8 */                                            \
3559          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3560          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3561          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3562          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3563          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3564          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3565          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3566          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3567          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3568          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3569          "mr 11,%1\n\t"                                           \
3570          "mr %0,3\n\t"                                            \
3571          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3572          VG_CONTRACT_FRAME_BY(144)                                \
3573          VG_CONTRACT_FRAME_BY(512)                                \
3574          : /*out*/   "=r" (_res)                                  \
3575          : /*in*/    "r" (&_argvec[2])                            \
3576          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3577       );                                                          \
3578       lval = (__typeof__(lval)) _res;                             \
3579    } while (0)
3580
3581 #endif /* PLAT_ppc64_aix5 */
3582
3583
3584 /* ------------------------------------------------------------------ */
3585 /* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS.               */
3586 /*                                                                    */
3587 /* ------------------------------------------------------------------ */
3588
3589 /* Some request codes.  There are many more of these, but most are not
3590    exposed to end-user view.  These are the public ones, all of the
3591    form 0x1000 + small_number.
3592
3593    Core ones are in the range 0x00000000--0x0000ffff.  The non-public
3594    ones start at 0x2000.
3595 */
3596
3597 /* These macros are used by tools -- they must be public, but don't
3598    embed them into other programs. */
3599 #define VG_USERREQ_TOOL_BASE(a,b) \
3600    ((unsigned int)(((a)&0xff) << 24 | ((b)&0xff) << 16))
3601 #define VG_IS_TOOL_USERREQ(a, b, v) \
3602    (VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000))
3603
3604 /* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !!
3605    This enum comprises an ABI exported by Valgrind to programs
3606    which use client requests.  DO NOT CHANGE THE ORDER OF THESE
3607    ENTRIES, NOR DELETE ANY -- add new ones at the end. */
3608 typedef
3609    enum { VG_USERREQ__RUNNING_ON_VALGRIND  = 0x1001,
3610           VG_USERREQ__DISCARD_TRANSLATIONS = 0x1002,
3611
3612           /* These allow any function to be called from the simulated
3613              CPU but run on the real CPU.  Nb: the first arg passed to
3614              the function is always the ThreadId of the running
3615              thread!  So CLIENT_CALL0 actually requires a 1 arg
3616              function, etc. */
3617           VG_USERREQ__CLIENT_CALL0 = 0x1101,
3618           VG_USERREQ__CLIENT_CALL1 = 0x1102,
3619           VG_USERREQ__CLIENT_CALL2 = 0x1103,
3620           VG_USERREQ__CLIENT_CALL3 = 0x1104,
3621
3622           /* Can be useful in regression testing suites -- eg. can
3623              send Valgrind's output to /dev/null and still count
3624              errors. */
3625           VG_USERREQ__COUNT_ERRORS = 0x1201,
3626
3627           /* These are useful and can be interpreted by any tool that
3628              tracks malloc() et al, by using vg_replace_malloc.c. */
3629           VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301,
3630           VG_USERREQ__FREELIKE_BLOCK   = 0x1302,
3631           /* Memory pool support. */
3632           VG_USERREQ__CREATE_MEMPOOL   = 0x1303,
3633           VG_USERREQ__DESTROY_MEMPOOL  = 0x1304,
3634           VG_USERREQ__MEMPOOL_ALLOC    = 0x1305,
3635           VG_USERREQ__MEMPOOL_FREE     = 0x1306,
3636           VG_USERREQ__MEMPOOL_TRIM     = 0x1307,
3637           VG_USERREQ__MOVE_MEMPOOL     = 0x1308,
3638           VG_USERREQ__MEMPOOL_CHANGE   = 0x1309,
3639           VG_USERREQ__MEMPOOL_EXISTS   = 0x130a,
3640
3641           /* Allow printfs to valgrind log. */
3642           VG_USERREQ__PRINTF           = 0x1401,
3643           VG_USERREQ__PRINTF_BACKTRACE = 0x1402,
3644
3645           /* Stack support. */
3646           VG_USERREQ__STACK_REGISTER   = 0x1501,
3647           VG_USERREQ__STACK_DEREGISTER = 0x1502,
3648           VG_USERREQ__STACK_CHANGE     = 0x1503,
3649
3650           /* Wine support */
3651           VG_USERREQ__LOAD_PDB_DEBUGINFO = 0x1601
3652    } Vg_ClientRequest;
3653
3654 #if !defined(__GNUC__)
3655 #  define __extension__ /* */
3656 #endif
3657
3658 /* Returns the number of Valgrinds this code is running under.  That
3659    is, 0 if running natively, 1 if running under Valgrind, 2 if
3660    running under Valgrind which is running under another Valgrind,
3661    etc. */
3662 #define RUNNING_ON_VALGRIND  __extension__                        \
3663    ({unsigned int _qzz_res;                                       \
3664     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0 /* if not */,          \
3665                                VG_USERREQ__RUNNING_ON_VALGRIND,   \
3666                                0, 0, 0, 0, 0);                    \
3667     _qzz_res;                                                     \
3668    })
3669
3670
3671 /* Discard translation of code in the range [_qzz_addr .. _qzz_addr +
3672    _qzz_len - 1].  Useful if you are debugging a JITter or some such,
3673    since it provides a way to make sure valgrind will retranslate the
3674    invalidated area.  Returns no value. */
3675 #define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len)         \
3676    {unsigned int _qzz_res;                                        \
3677     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3678                                VG_USERREQ__DISCARD_TRANSLATIONS,  \
3679                                _qzz_addr, _qzz_len, 0, 0, 0);     \
3680    }
3681
3682
3683 /* These requests are for getting Valgrind itself to print something.
3684    Possibly with a backtrace.  This is a really ugly hack.  The return value
3685    is the number of characters printed, excluding the "**<pid>** " part at the
3686    start and the backtrace (if present). */
3687
3688 #if defined(NVALGRIND)
3689
3690 #  define VALGRIND_PRINTF(...)
3691 #  define VALGRIND_PRINTF_BACKTRACE(...)
3692
3693 #else /* NVALGRIND */
3694
3695 /* Modern GCC will optimize the static routine out if unused,
3696    and unused attribute will shut down warnings about it.  */
3697 static int VALGRIND_PRINTF(const char *format, ...)
3698    __attribute__((format(__printf__, 1, 2), __unused__));
3699 static int
3700 VALGRIND_PRINTF(const char *format, ...)
3701 {
3702    unsigned long _qzz_res;
3703    va_list vargs;
3704    va_start(vargs, format);
3705    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, VG_USERREQ__PRINTF,
3706                               (unsigned long)format, (unsigned long)vargs,
3707                               0, 0, 0);
3708    va_end(vargs);
3709    return (int)_qzz_res;
3710 }
3711
3712 static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
3713    __attribute__((format(__printf__, 1, 2), __unused__));
3714 static int
3715 VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
3716 {
3717    unsigned long _qzz_res;
3718    va_list vargs;
3719    va_start(vargs, format);
3720    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, VG_USERREQ__PRINTF_BACKTRACE,
3721                               (unsigned long)format, (unsigned long)vargs,
3722                               0, 0, 0);
3723    va_end(vargs);
3724    return (int)_qzz_res;
3725 }
3726
3727 #endif /* NVALGRIND */
3728
3729
3730 /* These requests allow control to move from the simulated CPU to the
3731    real CPU, calling an arbitary function.
3732
3733    Note that the current ThreadId is inserted as the first argument.
3734    So this call:
3735
3736      VALGRIND_NON_SIMD_CALL2(f, arg1, arg2)
3737
3738    requires f to have this signature:
3739
3740      Word f(Word tid, Word arg1, Word arg2)
3741
3742    where "Word" is a word-sized type.
3743
3744    Note that these client requests are not entirely reliable.  For example,
3745    if you call a function with them that subsequently calls printf(),
3746    there's a high chance Valgrind will crash.  Generally, your prospects of
3747    these working are made higher if the called function does not refer to
3748    any global variables, and does not refer to any libc or other functions
3749    (printf et al).  Any kind of entanglement with libc or dynamic linking is
3750    likely to have a bad outcome, for tricky reasons which we've grappled
3751    with a lot in the past.
3752 */
3753 #define VALGRIND_NON_SIMD_CALL0(_qyy_fn)                          \
3754    __extension__                                                  \
3755    ({unsigned long _qyy_res;                                      \
3756     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
3757                                VG_USERREQ__CLIENT_CALL0,          \
3758                                _qyy_fn,                           \
3759                                0, 0, 0, 0);                       \
3760     _qyy_res;                                                     \
3761    })
3762
3763 #define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1)               \
3764    __extension__                                                  \
3765    ({unsigned long _qyy_res;                                      \
3766     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
3767                                VG_USERREQ__CLIENT_CALL1,          \
3768                                _qyy_fn,                           \
3769                                _qyy_arg1, 0, 0, 0);               \
3770     _qyy_res;                                                     \
3771    })
3772
3773 #define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2)    \
3774    __extension__                                                  \
3775    ({unsigned long _qyy_res;                                      \
3776     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
3777                                VG_USERREQ__CLIENT_CALL2,          \
3778                                _qyy_fn,                           \
3779                                _qyy_arg1, _qyy_arg2, 0, 0);       \
3780     _qyy_res;                                                     \
3781    })
3782
3783 #define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \
3784    __extension__                                                  \
3785    ({unsigned long _qyy_res;                                      \
3786     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
3787                                VG_USERREQ__CLIENT_CALL3,          \
3788                                _qyy_fn,                           \
3789                                _qyy_arg1, _qyy_arg2,              \
3790                                _qyy_arg3, 0);                     \
3791     _qyy_res;                                                     \
3792    })
3793
3794
3795 /* Counts the number of errors that have been recorded by a tool.  Nb:
3796    the tool must record the errors with VG_(maybe_record_error)() or
3797    VG_(unique_error)() for them to be counted. */
3798 #define VALGRIND_COUNT_ERRORS                                     \
3799    __extension__                                                  \
3800    ({unsigned int _qyy_res;                                       \
3801     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
3802                                VG_USERREQ__COUNT_ERRORS,          \
3803                                0, 0, 0, 0, 0);                    \
3804     _qyy_res;                                                     \
3805    })
3806
3807 /* Several Valgrind tools (Memcheck, Massif, Helgrind, DRD) rely on knowing
3808    when heap blocks are allocated in order to give accurate results.  This
3809    happens automatically for the standard allocator functions such as
3810    malloc(), calloc(), realloc(), memalign(), new, new[], free(), delete,
3811    delete[], etc.
3812
3813    But if your program uses a custom allocator, this doesn't automatically
3814    happen, and Valgrind will not do as well.  For example, if you allocate
3815    superblocks with mmap() and then allocates chunks of the superblocks, all
3816    Valgrind's observations will be at the mmap() level and it won't know that
3817    the chunks should be considered separate entities.  In Memcheck's case,
3818    that means you probably won't get heap block overrun detection (because
3819    there won't be redzones marked as unaddressable) and you definitely won't
3820    get any leak detection.
3821
3822    The following client requests allow a custom allocator to be annotated so
3823    that it can be handled accurately by Valgrind.
3824
3825    VALGRIND_MALLOCLIKE_BLOCK marks a region of memory as having been allocated
3826    by a malloc()-like function.  For Memcheck (an illustrative case), this
3827    does two things:
3828
3829    - It records that the block has been allocated.  This means any addresses
3830      within the block mentioned in error messages will be
3831      identified as belonging to the block.  It also means that if the block
3832      isn't freed it will be detected by the leak checker.
3833
3834    - It marks the block as being addressable and undefined (if 'is_zeroed' is
3835      not set), or addressable and defined (if 'is_zeroed' is set).  This
3836      controls how accesses to the block by the program are handled.
3837
3838    'addr' is the start of the usable block (ie. after any
3839    redzone), 'sizeB' is its size.  'rzB' is the redzone size if the allocator
3840    can apply redzones -- these are blocks of padding at the start and end of
3841    each block.  Adding redzones is recommended as it makes it much more likely
3842    Valgrind will spot block overruns.  `is_zeroed' indicates if the memory is
3843    zeroed (or filled with another predictable value), as is the case for
3844    calloc().
3845
3846    VALGRIND_MALLOCLIKE_BLOCK should be put immediately after the point where a
3847    heap block -- that will be used by the client program -- is allocated.
3848    It's best to put it at the outermost level of the allocator if possible;
3849    for example, if you have a function my_alloc() which calls
3850    internal_alloc(), and the client request is put inside internal_alloc(),
3851    stack traces relating to the heap block will contain entries for both
3852    my_alloc() and internal_alloc(), which is probably not what you want.
3853
3854    For Memcheck users: if you use VALGRIND_MALLOCLIKE_BLOCK to carve out
3855    custom blocks from within a heap block, B, that has been allocated with
3856    malloc/calloc/new/etc, then block B will be *ignored* during leak-checking
3857    -- the custom blocks will take precedence.
3858
3859    VALGRIND_FREELIKE_BLOCK is the partner to VALGRIND_MALLOCLIKE_BLOCK.  For
3860    Memcheck, it does two things:
3861
3862    - It records that the block has been deallocated.  This assumes that the
3863      block was annotated as having been allocated via
3864      VALGRIND_MALLOCLIKE_BLOCK.  Otherwise, an error will be issued.
3865
3866    - It marks the block as being unaddressable.
3867
3868    VALGRIND_FREELIKE_BLOCK should be put immediately after the point where a
3869    heap block is deallocated.
3870
3871    In many cases, these two client requests will not be enough to get your
3872    allocator working well with Memcheck.  More specifically, if your allocator
3873    writes to freed blocks in any way then a VALGRIND_MAKE_MEM_UNDEFINED call
3874    will be necessary to mark the memory as addressable just before the zeroing
3875    occurs, otherwise you'll get a lot of invalid write errors.  For example,
3876    you'll need to do this if your allocator recycles freed blocks, but it
3877    zeroes them before handing them back out (via VALGRIND_MALLOCLIKE_BLOCK).
3878    Alternatively, if your allocator reuses freed blocks for allocator-internal
3879    data structures, VALGRIND_MAKE_MEM_UNDEFINED calls will also be necessary.
3880
3881    Really, what's happening is a blurring of the lines between the client
3882    program and the allocator... after VALGRIND_FREELIKE_BLOCK is called, the
3883    memory should be considered unaddressable to the client program, but the
3884    allocator knows more than the rest of the client program and so may be able
3885    to safely access it.  Extra client requests are necessary for Valgrind to
3886    understand the distinction between the allocator and the rest of the
3887    program.
3888
3889    Note: there is currently no VALGRIND_REALLOCLIKE_BLOCK client request;  it
3890    has to be emulated with MALLOCLIKE/FREELIKE and memory copying.
3891
3892    Ignored if addr == 0.
3893 */
3894 #define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)    \
3895    {unsigned int _qzz_res;                                        \
3896     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3897                                VG_USERREQ__MALLOCLIKE_BLOCK,      \
3898                                addr, sizeB, rzB, is_zeroed, 0);   \
3899    }
3900
3901 /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
3902    Ignored if addr == 0.
3903 */
3904 #define VALGRIND_FREELIKE_BLOCK(addr, rzB)                        \
3905    {unsigned int _qzz_res;                                        \
3906     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3907                                VG_USERREQ__FREELIKE_BLOCK,        \
3908                                addr, rzB, 0, 0, 0);               \
3909    }
3910
3911 /* Create a memory pool. */
3912 #define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed)             \
3913    {unsigned int _qzz_res;                                        \
3914     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3915                                VG_USERREQ__CREATE_MEMPOOL,        \
3916                                pool, rzB, is_zeroed, 0, 0);       \
3917    }
3918
3919 /* Destroy a memory pool. */
3920 #define VALGRIND_DESTROY_MEMPOOL(pool)                            \
3921    {unsigned int _qzz_res;                                        \
3922     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3923                                VG_USERREQ__DESTROY_MEMPOOL,       \
3924                                pool, 0, 0, 0, 0);                 \
3925    }
3926
3927 /* Associate a piece of memory with a memory pool. */
3928 #define VALGRIND_MEMPOOL_ALLOC(pool, addr, size)                  \
3929    {unsigned int _qzz_res;                                        \
3930     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3931                                VG_USERREQ__MEMPOOL_ALLOC,         \
3932                                pool, addr, size, 0, 0);           \
3933    }
3934
3935 /* Disassociate a piece of memory from a memory pool. */
3936 #define VALGRIND_MEMPOOL_FREE(pool, addr)                         \
3937    {unsigned int _qzz_res;                                        \
3938     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3939                                VG_USERREQ__MEMPOOL_FREE,          \
3940                                pool, addr, 0, 0, 0);              \
3941    }
3942
3943 /* Disassociate any pieces outside a particular range. */
3944 #define VALGRIND_MEMPOOL_TRIM(pool, addr, size)                   \
3945    {unsigned int _qzz_res;                                        \
3946     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3947                                VG_USERREQ__MEMPOOL_TRIM,          \
3948                                pool, addr, size, 0, 0);           \
3949    }
3950
3951 /* Resize and/or move a piece associated with a memory pool. */
3952 #define VALGRIND_MOVE_MEMPOOL(poolA, poolB)                       \
3953    {unsigned int _qzz_res;                                        \
3954     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3955                                VG_USERREQ__MOVE_MEMPOOL,          \
3956                                poolA, poolB, 0, 0, 0);            \
3957    }
3958
3959 /* Resize and/or move a piece associated with a memory pool. */
3960 #define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size)         \
3961    {unsigned int _qzz_res;                                        \
3962     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3963                                VG_USERREQ__MEMPOOL_CHANGE,        \
3964                                pool, addrA, addrB, size, 0);      \
3965    }
3966
3967 /* Return 1 if a mempool exists, else 0. */
3968 #define VALGRIND_MEMPOOL_EXISTS(pool)                             \
3969    __extension__                                                  \
3970    ({unsigned int _qzz_res;                                       \
3971     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3972                                VG_USERREQ__MEMPOOL_EXISTS,        \
3973                                pool, 0, 0, 0, 0);                 \
3974     _qzz_res;                                                     \
3975    })
3976
3977 /* Mark a piece of memory as being a stack. Returns a stack id. */
3978 #define VALGRIND_STACK_REGISTER(start, end)                       \
3979    __extension__                                                  \
3980    ({unsigned int _qzz_res;                                       \
3981     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3982                                VG_USERREQ__STACK_REGISTER,        \
3983                                start, end, 0, 0, 0);              \
3984     _qzz_res;                                                     \
3985    })
3986
3987 /* Unmark the piece of memory associated with a stack id as being a
3988    stack. */
3989 #define VALGRIND_STACK_DEREGISTER(id)                             \
3990    {unsigned int _qzz_res;                                        \
3991     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3992                                VG_USERREQ__STACK_DEREGISTER,      \
3993                                id, 0, 0, 0, 0);                   \
3994    }
3995
3996 /* Change the start and end address of the stack id. */
3997 #define VALGRIND_STACK_CHANGE(id, start, end)                     \
3998    {unsigned int _qzz_res;                                        \
3999     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4000                                VG_USERREQ__STACK_CHANGE,          \
4001                                id, start, end, 0, 0);             \
4002    }
4003
4004 /* Load PDB debug info for Wine PE image_map. */
4005 #define VALGRIND_LOAD_PDB_DEBUGINFO(fd, ptr, total_size, delta)   \
4006    {unsigned int _qzz_res;                                        \
4007     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4008                                VG_USERREQ__LOAD_PDB_DEBUGINFO,    \
4009                                fd, ptr, total_size, delta, 0);    \
4010    }
4011
4012
4013 #undef PLAT_x86_linux
4014 #undef PLAT_amd64_linux
4015 #undef PLAT_ppc32_linux
4016 #undef PLAT_ppc64_linux
4017 #undef PLAT_ppc32_aix5
4018 #undef PLAT_ppc64_aix5
4019
4020 #endif   /* __VALGRIND_H */