/*------------------------------------------------------------------------------------------------*/
/* Copyright (C) by the DBCSR developers group - All rights reserved                              */
/* This file is part of the DBCSR library.                                                        */
/*                                                                                                */
/* For information on the license, see the LICENSE file.                                          */
/* For further information please visit https://dbcsr.cp2k.org                                    */
/* SPDX-License-Identifier: GPL-2.0+                                                              */
/*------------------------------------------------------------------------------------------------*/
#if defined(__OPENCL)
#  include "opencl_libsmm.h"
/* Header opencl_kernels.h is generated by the build system using acc_opencl.sh */
#  include "opencl_kernels.h"
#  include "../../acc_bench.h"
#  include <ctype.h>

#  if LIBXSMM_VERSION4(1, 17, 0, 0) < LIBXSMM_VERSION_NUMBER
#    define OPENCL_LIBSMM_GEMM_BATCH(IPREC, OPREC, TRANSA, TRANSB, M, N, K, ALPHA, A, LDA, STRIDE_A, B, LDB, STRIDE_B, BETA, C, \
      LDC, STRIDE_C, INDEX_STRIDE, INDEX_BASE, BATCHSIZE) \
      OPENCL_LIBSMM_USEOMP(libxsmm_gemm_batch) \
      (IPREC, OPREC, TRANSA, TRANSB, M, N, K, ALPHA, A, LDA, STRIDE_A, B, LDB, STRIDE_B, BETA, C, LDC, STRIDE_C, INDEX_STRIDE, \
        INDEX_BASE, BATCHSIZE, 0 /*batchcheck*/)
#  else
#    define OPENCL_LIBSMM_GEMM_BATCH(IPREC, OPREC, TRANSA, TRANSB, M, N, K, ALPHA, A, LDA, STRIDE_A, B, LDB, STRIDE_B, BETA, C, \
      LDC, STRIDE_C, INDEX_STRIDE, INDEX_BASE, BATCHSIZE) \
      OPENCL_LIBSMM_USEOMP(libxsmm_gemm_batch) \
      ((libxsmm_gemm_precision)(IPREC), (libxsmm_gemm_precision)(OPREC), TRANSA, TRANSB, M, N, K, ALPHA, A, LDA, B, LDB, BETA, C, \
        LDC, INDEX_BASE, INDEX_STRIDE, STRIDE_A, STRIDE_B, STRIDE_C, BATCHSIZE)
#  endif

#  if defined(_OPENMP) && !defined(__DBCSR_ACC)
#    define OPENCL_LIBSMM_USEOMP(FUNC) LIBXSMM_USEOMP(FUNC)
#  else
#    define OPENCL_LIBSMM_USEOMP(FUNC) (FUNC)
#  endif

#  if !defined(OPENCL_LIBSMM_VALIDATE_TRANS) && defined(OPENCL_LIBSMM_VALIDATE) && \
    (1 < OPENCL_LIBSMM_VALIDATE || 0 > OPENCL_LIBSMM_VALIDATE)
#    define OPENCL_LIBSMM_VALIDATE_TRANS
#  endif
#  if !defined(OPENCL_LIBSMM_VALIDATE_SMM) && defined(OPENCL_LIBSMM_VALIDATE)
#    define OPENCL_LIBSMM_VALIDATE_SMM
#  endif
#  if !defined(OPENCL_LIBSMM_VALIDATE_EXIT) && defined(OPENCL_LIBSMM_VALIDATE) && 1
#    define OPENCL_LIBSMM_VALIDATE_EXIT
#  endif
#  if !defined(OPENCL_LIBSMM_KERNELNAME_TRANS)
#    define OPENCL_LIBSMM_KERNELNAME_TRANS "trans"
#  endif
#  if !defined(OPENCL_LIBSMM_KERNELNAME_SMM)
#    define OPENCL_LIBSMM_KERNELNAME_SMM "smm"
#  endif
#  if !defined(OPENCL_LIBSMM_NLOCKS_TRANS)
#    define OPENCL_LIBSMM_NLOCKS_TRANS 16
#  endif
#  if !defined(OPENCL_LIBSMM_NLOCKS_SMM)
#    define OPENCL_LIBSMM_NLOCKS_SMM 16
#  endif
#  if !defined(OPENCL_LIBSMM_CMEM) && 1
#    define OPENCL_LIBSMM_CMEM
#  endif
/* default: decompose C-matrix into column-vectors (BMxBN) */
#  if !defined(OPENCL_LIBSMM_DEFAULT_BM)
#    define OPENCL_LIBSMM_DEFAULT_BM INT_MAX
#  endif
#  if !defined(OPENCL_LIBSMM_DEFAULT_BN)
#    define OPENCL_LIBSMM_DEFAULT_BN 1
#  endif
#  if !defined(OPENCL_LIBSMM_DEFAULT_BK)
#    if 1
#      define OPENCL_LIBSMM_DEFAULT_BK INT_MAX
#    else
#      define OPENCL_LIBSMM_DEFAULT_BK 1
#    endif
#  endif
#  if !defined(OPENCL_LIBSMM_DEFAULT_BS)
#    define OPENCL_LIBSMM_DEFAULT_BS 8
#  endif
#  if !defined(OPENCL_LIBSMM_BS_MIN) && 1
#    define OPENCL_LIBSMM_BS_MIN 32
#  endif
#  if !defined(OPENCL_LIBSMM_SMM_S)
#    define OPENCL_LIBSMM_SMM_S 64
#  endif
#  if !defined(OPENCL_LIBSMM_VLEN)
#    define OPENCL_LIBSMM_VLEN 32
#  endif
#  if !defined(OPENCL_LIBSMM_VMIN)
#    define OPENCL_LIBSMM_VMIN 8
#  endif

/* approximate arithmetic intensity for SMMs like C += Ai * Bi (beta=1) */
#  define OPENCL_LIBSMM_AI(M, N, K, TYPESIZE) ((2.0 * (M) * (N) * (K)) / ((TYPESIZE) * (K) * ((M) + (N))))

#  define OPENCL_LIBSMM_TYPESIZE(TYPEID) \
    (dbcsr_type_real_8 == (TYPEID) ? ((int)sizeof(double)) : (dbcsr_type_real_4 == (TYPEID) ? ((int)sizeof(float)) : 0 /*unknown*/))

#  define OPENCL_LIBSMM_ISORT(IARR, SIZE) \
    { \
      int opencl_libsmm_isort_i_ = 0; \
      for (; opencl_libsmm_isort_i_ < ((int)(SIZE)-1); ++opencl_libsmm_isort_i_) { \
        int opencl_libsmm_isort_j_ = opencl_libsmm_isort_i_ + 2; \
        int opencl_libsmm_isort_k_ = opencl_libsmm_isort_i_ + 1; \
        for (; opencl_libsmm_isort_j_ < ((int)(SIZE)); ++opencl_libsmm_isort_j_) { \
          if ((IARR)[opencl_libsmm_isort_j_] < (IARR)[opencl_libsmm_isort_k_]) { \
            opencl_libsmm_isort_k_ = opencl_libsmm_isort_j_; \
          } \
        } \
        if ((IARR)[opencl_libsmm_isort_k_] < (IARR)[opencl_libsmm_isort_i_]) { \
          LIBXSMM_ISWAP((IARR)[opencl_libsmm_isort_i_], (IARR)[opencl_libsmm_isort_k_]); \
        } \
      } \
    }


#  if defined(__cplusplus)
extern "C" {
#  endif

/* maintain GFLOPS/AI ratios for performance estimates and suitability */
double opencl_libsmm_shst, opencl_libsmm_dhst, opencl_libsmm_sacc, opencl_libsmm_dacc;
/* track initialization status of LIBSMM */
int opencl_libsmm_initialized;


int opencl_libsmm_use_cmem(cl_device_id device) {
#  if defined(OPENCL_LIBSMM_CMEM)
  int result = EXIT_SUCCESS;
  cl_ulong size_maxalloc = 1, size_maxcmem = 0;
  ACC_OPENCL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_MEM_ALLOC_SIZE, sizeof(cl_ulong), &size_maxalloc, NULL),
    "retrieve maximum size of memory allocation", result);
  ACC_OPENCL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_CONSTANT_BUFFER_SIZE, sizeof(cl_ulong), &size_maxcmem, NULL),
    "retrieve maximum size of constant buffer", result);
  return (EXIT_SUCCESS == result ? (size_maxalloc <= size_maxcmem ? EXIT_SUCCESS : EXIT_FAILURE) : result);
#  else
  LIBXSMM_UNUSED(device);
  return EXIT_FAILURE;
#  endif
}


#  if defined(_DEBUG) && defined(OPENCL_LIBSMM_VALIDATE) && (0 != OPENCL_LIBSMM_VALIDATE)
void opencl_libsmm_print_matrix(FILE* ostream, const char* label, libsmm_acc_data_t type, const void* mat, int m, int n) {
  int i, j;
  const char* const s = (NULL != label ? label : "");
  const int len = (int)strlen(s);
  for (i = 0; i < m; ++i) {
    if (0 < i) {
      fprintf(ostream, "%*s", len, " ");
    }
    else {
      fprintf(ostream, "%s", s);
    }
    for (j = 0; j < n; ++j) {
      switch (type) {
        case dbcsr_type_real_8: fprintf(ostream, "%.2f ", ((double*)mat)[i * n + j]); break;
        case dbcsr_type_real_4: fprintf(ostream, "%.2f ", ((float*)mat)[i * n + j]); break;
        default: fprintf(ostream, "? ");
      }
    }
    fprintf(ostream, "\n");
  }
}
#  endif


int opencl_libsmm_write_trans_params(FILE* stream, int only_key, const opencl_libsmm_transkey_t* key,
  const opencl_libsmm_trans_t* config, const char* delim, const char* begin, const char* close) {
  int result = 0;
  if (NULL != stream) {
    const char d = (NULL == delim ? *ACC_OPENCL_DELIMS : *delim);
    if (NULL != key || 0 == only_key) result += fprintf(stream, "%c", NULL == begin ? '{' : *begin);
    if (NULL != config) {
      if (NULL != key) {
        result += fprintf(stream, "%i%c%i%c%i", (int)key->type, d, key->m, d, key->n);
        /*if (0 == only_key) result += fprintf(stream, "%c", d);*/
      }
    }
    else {
      if (NULL != key) {
        result += fprintf(stream, "t%cm%cn", d, d);
        /*if (0 == only_key) result += fprintf(stream, "%c", d);*/
      }
    }
    if (NULL != key || 0 == only_key) result += fprintf(stream, "%c", NULL == close ? '}' : *close);
  }
  else result = -1;
  assert(0 < result);
  return result;
}


int opencl_libsmm_write_smm_params(FILE* stream, int only_key, const opencl_libsmm_smmkey_t* key, const opencl_libsmm_smm_t* config,
  const char* delim, const char* begin, const char* close) {
  int result = 0;
  if (NULL != stream) {
    const char d = (NULL == delim ? *ACC_OPENCL_DELIMS : *delim);
    if (NULL != key || 0 == only_key) result += fprintf(stream, "%c", NULL == begin ? '{' : *begin);
    if (NULL != config) {
      if (NULL != key) {
        result += fprintf(stream, "%i%c%i%c%i%c%i", (int)key->type, d, key->m, d, key->n, d, key->k);
        if (0 == only_key) result += fprintf(stream, "%c ", d);
      }
      if (0 == only_key) {
        result += fprintf(stream, "%i%c%i%c%i%c%i%c %i%c%i%c %i%c%i%c%i%c %i%c%i%c %i%c%i%c%i%c%i", config->bs, d, config->bm, d,
          config->bn, d, config->bk, d, config->ws, d, config->wg, d, config->lu, d, config->nz, d, config->al, d, config->tb, d,
          config->tc, d, config->ap, d, config->aa, d, config->ab, d, config->ac);
      }
    }
    else {
      if (NULL != key) {
        result += fprintf(stream, "t%cm%cn%ck", d, d, d);
        if (0 == only_key) result += fprintf(stream, "%c ", d);
      }
      if (0 == only_key) {
        result += fprintf(
          stream, "bs%cbm%cbn%cbk%c ws%cwg%c lu%cnz%cal%c tb%ctc%c ap%caa%cab%cac", d, d, d, d, d, d, d, d, d, d, d, d, d, d);
      }
    }
    if (NULL != key || 0 == only_key) result += fprintf(stream, "%c", NULL == close ? '}' : *close);
  }
  else result = -1;
  assert(0 < result);
  return result;
}


int opencl_libsmm_read_smm_params(
  char* parambuf, opencl_libsmm_smmkey_t* key, opencl_libsmm_smm_t* value, opencl_libsmm_perfest_t* perfest, char* device) {
  const char* const end = parambuf + strlen(parambuf); /* before strtok */
  char* s = strtok(parambuf, ACC_OPENCL_DELIMS);
  int result = EXIT_SUCCESS, i = 0, ivalue, consumed = 0, c = 0;
  const int opt_consumed = (NULL != perfest ? 2 : 0) + (NULL != device ? 1 : 0);
  const int max_consumed = opt_consumed + 19;
  double gflops;
  assert(NULL != key && NULL != value);
  for (; NULL != s;
       ++i, s = (c != consumed ? ((s + 1) < end ? strtok((s + 1) + strlen(s), ACC_OPENCL_DELIMS) : NULL) : s), c = consumed)
  {
    switch (i) {
      case 0:
        if (NULL != device && 1 == sscanf(s, "%[^" ACC_OPENCL_DELIMS "]", device)) {
          ++consumed; /* optional device name */
        }
        break;
      case 1:
        if (1 == sscanf(s, "%i", &ivalue)) {
          key->type = (libsmm_acc_data_t)ivalue;
          ++consumed;
        }
        break;
      case 2:
        if (1 == sscanf(s, "%i", &ivalue) && 0 < ivalue) {
          key->m = ivalue;
          ++consumed;
        }
        break;
      case 3:
        if (1 == sscanf(s, "%i", &ivalue) && 0 < ivalue) {
          key->n = ivalue;
          ++consumed;
        }
        break;
      case 4:
        if (1 == sscanf(s, "%i", &ivalue) && 0 < ivalue) {
          key->k = ivalue;
          ++consumed;
        }
        break;
      case 5:
        if (NULL != perfest && 1 == sscanf(s, "%i", &ivalue)) {
          value->s = ivalue;
          ++consumed; /* optional "S" param */
        }
        break;
      case 6:
        if (NULL != perfest && 1 == sscanf(s, "%lf", &gflops) && 0 <= gflops) {
          value->gflops = gflops;
          ++consumed; /* optional "GFLOPS" param */
        }
        break;
      case 7:
        if (1 == sscanf(s, "%i", &ivalue)) {
          value->bs = ivalue;
          ++consumed;
        }
        break;
      case 8:
        if (1 == sscanf(s, "%i", &ivalue)) {
          value->bm = ivalue;
          ++consumed;
        }
        break;
      case 9:
        if (1 == sscanf(s, "%i", &ivalue)) {
          value->bn = ivalue;
          ++consumed;
        }
        break;
      case 10:
        if (1 == sscanf(s, "%i", &ivalue)) {
          value->bk = ivalue;
          ++consumed;
        }
        break;
      case 11:
        if (1 == sscanf(s, "%i", &ivalue)) {
          value->ws = ivalue;
          ++consumed;
        }
        break;
      case 12:
        if (1 == sscanf(s, "%i", &ivalue)) {
          value->wg = ivalue;
          ++consumed;
        }
        break;
      case 13:
        if (1 == sscanf(s, "%i", &ivalue)) {
          value->lu = ivalue;
          ++consumed;
        }
        break;
      case 14:
        if (1 == sscanf(s, "%i", &ivalue)) {
          value->nz = ivalue;
          ++consumed;
        }
        break;
      case 15:
        if (1 == sscanf(s, "%i", &ivalue)) {
          value->al = ivalue;
          ++consumed;
        }
        break;
      case 16:
        if (1 == sscanf(s, "%i", &ivalue)) {
          value->tb = ivalue;
          ++consumed;
        }
        break;
      case 17:
        if (1 == sscanf(s, "%i", &ivalue)) {
          value->tc = ivalue;
          ++consumed;
        }
        break;
      case 18:
        if (1 == sscanf(s, "%i", &ivalue)) {
          value->ap = ivalue;
          ++consumed;
        }
        break;
      case 19:
        if (1 == sscanf(s, "%i", &ivalue)) {
          value->aa = ivalue;
          ++consumed;
        }
        break;
      case 20:
        if (1 == sscanf(s, "%i", &ivalue)) {
          value->ab = ivalue;
          ++consumed;
        }
        break;
      case 21:
        if (1 == sscanf(s, "%i", &ivalue)) {
          value->ac = ivalue;
          ++consumed;
        }
        break;
      default: s = NULL; /* break */
    }
  }
  if (max_consumed == consumed) {
    if (NULL != perfest) {
      switch (key->type) {
        case dbcsr_type_real_8: {
          const double ratio = gflops / OPENCL_LIBSMM_AI(key->m, key->n, key->k, sizeof(double));
#  if LIBXSMM_VERSION4(1, 17, 0, 0) < LIBXSMM_VERSION_NUMBER
          libxsmm_kahan_sum(log(ratio), &perfest->gf_ai_dratio_sumlog, &perfest->gf_ai_dratio_kahan);
#  else
          perfest->gf_ai_dratio_sumlog += log(ratio);
#  endif
          if (perfest->gf_ai_dratio_max < ratio) perfest->gf_ai_dratio_max = ratio;
          ++perfest->dcount;
        } break;
        case dbcsr_type_real_4: {
          const double ratio = gflops / OPENCL_LIBSMM_AI(key->m, key->n, key->k, sizeof(float));
#  if LIBXSMM_VERSION4(1, 17, 0, 0) < LIBXSMM_VERSION_NUMBER
          libxsmm_kahan_sum(log(ratio), &perfest->gf_ai_sratio_sumlog, &perfest->gf_ai_sratio_kahan);
#  else
          perfest->gf_ai_sratio_sumlog += log(ratio);
#  endif
          if (perfest->gf_ai_sratio_max < ratio) perfest->gf_ai_sratio_max = ratio;
          ++perfest->scount;
        } break;
        default: result = EXIT_FAILURE;
      }
    }
  }
  else result = EXIT_FAILURE;
  return result;
}


int libsmm_acc_init(void) {
#  if defined(_OPENMP)
  /* initialization/finalization is not meant to be thread-safe */
  int result = ((0 == omp_in_parallel() || /*main*/ 0 == omp_get_thread_num()) ? EXIT_SUCCESS : EXIT_FAILURE);
#  else
  int result = EXIT_SUCCESS;
#  endif
  /* multiple calls to libsmm_acc_init are not considered as an error */
  if (1 == LIBXSMM_ATOMIC_ADD_FETCH(&opencl_libsmm_initialized, 1, LIBXSMM_ATOMIC_RELAXED)) {
#  if !defined(__DBCSR_ACC)
    /* DBCSR shall call c_dbcsr_acc_init as well as libsmm_acc_init (since both interfaces are used).
     * Also, libsmm_acc_init may privately call c_dbcsr_acc_init (as it depends on the ACC interface).
     * The implementation of c_dbcsr_acc_init should hence be safe against "over initialization".
     * However, DBCSR only calls c_dbcsr_acc_init (and expects an implicit libsmm_acc_init).
     */
    if (EXIT_SUCCESS == result) result = c_dbcsr_acc_init();
#  endif
    libxsmm_init();
    if (EXIT_SUCCESS == result) {
      opencl_libsmm_perfest_t perfest;
      char* const env_params = getenv("OPENCL_LIBSMM_SMM_PARAMS");
      const char* const env_suitable = getenv("OPENCL_LIBSMM_SUITABLE");
#  if defined(OPENCL_LIBSMM_SUITABLE)
      const int suitable = (NULL == env_suitable ? 1 : ('0' != *env_suitable));
#  else
      const int suitable = (NULL == env_suitable ? 0 : ('0' != *env_suitable));
#  endif
      memset(&perfest, 0, sizeof(perfest));
      if (NULL == env_params || '0' != *env_params) {
        char buffer[ACC_OPENCL_BUFFERSIZE], bufname[ACC_OPENCL_BUFFERSIZE], control = '0';
#  if defined(OPENCL_LIBSMM_DEVICES)
        const int ndevices_params = (int)(sizeof(OPENCL_LIBSMM_DEVICES) / sizeof(*OPENCL_LIBSMM_DEVICES));
        unsigned int ntuned = 0;
#  endif
        opencl_libsmm_smm_t config;
        opencl_libsmm_smmkey_t key;
        LIBXSMM_MEMZERO127(&key); /* potentially heterogeneous key-data (alignment gaps) */
        if (NULL != env_params && '\0' != *env_params) { /* filename */
          FILE* const file = fopen(env_params, "r");
          if (NULL != file) {
            /* consume first line, check for device entry, and skip CSV header */
            if (NULL != fgets(buffer, ACC_OPENCL_BUFFERSIZE, file)) {
              char* const device = (NULL != LIBXSMM_STRISTR(buffer, "device") ? bufname : NULL);
              opencl_libsmm_perfest_t* const gflops = (NULL != LIBXSMM_STRISTR(buffer, "gflops") ? &perfest : NULL);
              while (NULL != fgets(buffer, ACC_OPENCL_BUFFERSIZE, file)) { /* read params from CSV-file */
                memset(&config, 0, sizeof(config));
                if (EXIT_SUCCESS == opencl_libsmm_read_smm_params(buffer, &key, &config, gflops, device)) {
                  opencl_libsmm_smm_t* config_init;
                  c_dbcsr_acc_opencl_config.devmatch = 0; /* disable device-match */
                  key.devuid = 0;
                  config_init = (opencl_libsmm_smm_t*)libxsmm_xdispatch(&key, sizeof(key));
                  if (NULL == config_init) {
                    if (NULL == libxsmm_xregister(&key, sizeof(key), sizeof(config), &config)) {
                      result = EXIT_FAILURE;
                      break;
                    }
#  if defined(OPENCL_LIBSMM_DEVICES)
                    else ++ntuned;
#  endif
                  }
                  else if (config_init->gflops < config.gflops) { /* update */
                    memcpy(config_init, &config, sizeof(config));
                  }
                }
                else {
                  if (0 != c_dbcsr_acc_opencl_config.verbosity) {
                    fprintf(stderr, "WARN LIBSMM: failed to load tuned parameters!\n");
                  }
                  break; /* invalid entry */
                }
              }
            }
            else { /* invalid header */
              result = EXIT_FAILURE;
            }
            fclose(file);
            control = '1';
          }
          else control = '2';
        }
#  if defined(OPENCL_LIBSMM_PARAMS_SMM) && defined(OPENCL_LIBSMM_DEVICES)
        if (EXIT_SUCCESS == result && '1' != control) {
          const char *line = OPENCL_LIBSMM_PARAMS_SMM, *next;
#    if LIBXSMM_VERSION4(1, 17, 0, 0) < LIBXSMM_VERSION_NUMBER
          cl_device_id active_id = NULL;
          unsigned int active_uid;
          int active_match = -1;
          if (EXIT_SUCCESS == c_dbcsr_acc_opencl_device(ACC_OPENCL_OMP_TID(), &active_id) &&
              EXIT_SUCCESS == c_dbcsr_acc_opencl_device_name(
                                active_id, bufname, ACC_OPENCL_BUFFERSIZE, NULL /*platform*/, 0 /*platform_maxlen*/) &&
              EXIT_SUCCESS == c_dbcsr_acc_opencl_device_uid(active_id, bufname, &active_uid))
          {
            int i = 0, best = 0;
            for (; i < ndevices_params; ++i) {
              const int score = libxsmm_strimatch(bufname, OPENCL_LIBSMM_DEVICES[i], NULL);
              unsigned int uid;
              if (best < score ||
                  ((best == score) &&
                    EXIT_SUCCESS == c_dbcsr_acc_opencl_device_uid(NULL /*device*/, OPENCL_LIBSMM_DEVICES[i], &uid) &&
                    uid == active_uid))
              {
                active_match = i;
                best = score;
              }
            }
          }
#    endif
          do {
            next = strchr(line, '\n');
            if (NULL != next && next < (line + ACC_OPENCL_BUFFERSIZE)) {
              const int len = next - line;
              memcpy(buffer, line, len);
              buffer[len] = '\0';
              memset(&config, 0, sizeof(config));
              if (EXIT_SUCCESS == opencl_libsmm_read_smm_params(/* read params from embedded params */
                                    buffer, &key, &config, &perfest, bufname /*consume name/id*/))
              {
                opencl_libsmm_smm_t* config_init;
                const int i = atoi(bufname);
                if (0 >= ndevices_params || 0 == c_dbcsr_acc_opencl_config.devmatch || 0 > i || ndevices_params <= i ||
                    EXIT_SUCCESS != c_dbcsr_acc_opencl_device_uid(NULL /*device*/, OPENCL_LIBSMM_DEVICES[i], &key.devuid))
                {
                  key.devuid = 0;
                }
                config_init = (opencl_libsmm_smm_t*)libxsmm_xdispatch(&key, sizeof(key));
                if (NULL == config_init) {
                  if (NULL == libxsmm_xregister(&key, sizeof(key), sizeof(config), &config)) {
                    result = EXIT_FAILURE;
                    break;
                  }
                  else ++ntuned;
                }
                else if (config_init->gflops < config.gflops) { /* update */
                  memcpy(config_init, &config, sizeof(config));
                }
#    if LIBXSMM_VERSION4(1, 17, 0, 0) < LIBXSMM_VERSION_NUMBER
                if (active_match == i && active_uid != key.devuid) {
                  key.devuid = active_uid;
                  config_init = (opencl_libsmm_smm_t*)libxsmm_xdispatch(&key, sizeof(key));
                  if (NULL == config_init && NULL != libxsmm_xregister(&key, sizeof(key), sizeof(config), &config)) {
                    static int info = 0;
                    if (0 == info && 0 != c_dbcsr_acc_opencl_config.verbosity &&
                        EXIT_SUCCESS == c_dbcsr_acc_opencl_device_name(
                                          active_id, bufname, ACC_OPENCL_BUFFERSIZE, NULL /*platform*/, 0 /*platform_maxlen*/))
                    {
                      fprintf(stderr, "INFO ACC/OpenCL: PARAMS of \"%s\" used for \"%s\"\n", OPENCL_LIBSMM_DEVICES[i], bufname);
                      info = 1;
                    }
                  }
                }
#    endif
              }
              else {
                if (0 != c_dbcsr_acc_opencl_config.verbosity) {
                  fprintf(stderr, "WARN LIBSMM: failed to load tuned parameters!\n");
                }
                break;
              }
              line = ++next;
            }
          } while (NULL != next);
        }
#  endif
        if (EXIT_SUCCESS == result) {
          if ('2' == control) { /* try interpreting OPENCL_LIBSMM_SMM_PARAMS as kernel parameters (not device-specific) */
            memset(&config, 0, sizeof(config));
            if (EXIT_SUCCESS == opencl_libsmm_read_smm_params(env_params, &key, &config, NULL /*perfest*/, NULL /*device*/)) {
              key.devuid = 0;
              if (NULL != libxsmm_xregister(&key, sizeof(key), sizeof(config), &config)) {
                c_dbcsr_acc_opencl_config.devmatch = 0; /* disable device-match */
#  if defined(OPENCL_LIBSMM_DEVICES)
                ntuned = LIBXSMM_MAX(ntuned, 1); /* no destinction of overridden or new */
#  endif
              }
              else result = EXIT_FAILURE;
            }
            else if (0 != c_dbcsr_acc_opencl_config.verbosity) { /* soft-error */
              fprintf(stderr, "WARN LIBSMM: failed to open parameter file!\n");
            }
          }
#  if defined(OPENCL_LIBSMM_DEVICES)
          if (0 != c_dbcsr_acc_opencl_config.verbosity && 0 != ntuned) {
            fprintf(stderr, "INFO ACC/OpenCL: PARAMS in %u set%s loaded targeting ", ntuned, 1 != ntuned ? "s" : "");
            if (0 != c_dbcsr_acc_opencl_config.devmatch) {
              fprintf(stderr, "%i device%s\n", ndevices_params, 1 != ndevices_params ? "s" : "");
              if (1 < c_dbcsr_acc_opencl_config.verbosity || 0 > c_dbcsr_acc_opencl_config.verbosity) {
                unsigned int i = 0;
                for (; i < (unsigned int)ndevices_params; ++i) {
                  fprintf(stderr, "INFO ACC/OpenCL: PARAMS -> \"%s\"\n", OPENCL_LIBSMM_DEVICES[i]);
                }
              }
            }
            else fprintf(stderr, "any device\n");
          }
#  endif
        }
      }
      if (0 != suitable && EXIT_SUCCESS == result) {
        const int stack_size = 30000, nrepeat = 100;
        const int nc = LIBXSMM_MAX(stack_size / 16, 1), na = 10 * nc, nb = 10 * nc;
        const int m = 8, n = 8, k = 8, mn = m * n, mk = m * k, kn = k * n;
        const size_t scratch_size = /*stack*/ stack_size * 3 * sizeof(int) +
                                    (/*a*/ na * mk + /*b*/ nb * kn + /*c*/ nc * mn) * /*max.typesize*/ sizeof(double) +
                                    3 * (LIBXSMM_ALIGNMENT - 1) /*alignments*/;
        void* const scratch = libxsmm_aligned_scratch(scratch_size, LIBXSMM_ALIGNMENT);
        int *const s = (int*)scratch, i;
        libxsmm_timer_tickint start;
        const char notrans = 'N';
        if (0 != perfest.scount && 0 < perfest.gf_ai_sratio_max) {
          if (NULL != scratch) {
            float* const a = (float*)LIBXSMM_UP2((uintptr_t)s + sizeof(int) * stack_size * 3, LIBXSMM_ALIGNMENT);
            float* const b = (float*)LIBXSMM_UP2((uintptr_t)a + sizeof(float) * na * mk, LIBXSMM_ALIGNMENT);
            float* const c = (float*)LIBXSMM_UP2((uintptr_t)b + sizeof(float) * nb * kn, LIBXSMM_ALIGNMENT);
            const float alpha = 1, beta = 1;
            init_stack(s, stack_size, 0 /*rnd_size*/, NULL /*rnd*/, mn, mk, kn, nc, na, nb);
#  if defined(_OPENMP)
#    pragma omp parallel
#  endif
            {
#  if defined(_OPENMP)
#    pragma omp for
#  endif
              for (i = 0; i < na; ++i) INIT_MAT(float, i + 42, &a[i * mk], m, k, 1.0 / (nc * na));
#  if defined(_OPENMP)
#    pragma omp for
#  endif
              for (i = 0; i < nb; ++i) INIT_MAT(float, i + 24, &b[i * kn], k, n, 1.0 / (nc * nb));
            }
            memset(c, 0, sizeof(float) * nc * mn);
            start = libxsmm_timer_tick();
            for (i = 0; i < nrepeat; ++i) {
              OPENCL_LIBSMM_GEMM_BATCH(LIBXSMM_DATATYPE_F32, LIBXSMM_DATATYPE_F32, &notrans, &notrans, m, n, k, &alpha, a,
                &m /*lda*/, s + 0 /*stride_a*/, b, &k /*ldb*/, s + 1 /*stride_b*/, &beta, c, &m /*ldc*/, s + 2 /*stride_c*/,
                sizeof(int) * 3, 1 /*index_base*/, stack_size);
            }
            opencl_libsmm_shst = 1E-9 * ((size_t)2 * m * n * k * stack_size * nrepeat) /
                                 (libxsmm_timer_duration(start, libxsmm_timer_tick()) * OPENCL_LIBSMM_AI(m, n, k, sizeof(float)));
          }
          opencl_libsmm_sacc = (/*sqrt(perfest.gf_ai_sratio_max **/
            exp(perfest.gf_ai_sratio_sumlog / perfest.scount));
        }
        if (0 != perfest.dcount && 0 < perfest.gf_ai_dratio_max) {
          if (NULL != scratch) {
            double* const a = (double*)LIBXSMM_UP2((uintptr_t)s + sizeof(int) * stack_size * 3, LIBXSMM_ALIGNMENT);
            double* const b = (double*)LIBXSMM_UP2((uintptr_t)a + sizeof(double) * na * mk, LIBXSMM_ALIGNMENT);
            double* const c = (double*)LIBXSMM_UP2((uintptr_t)b + sizeof(double) * nb * kn, LIBXSMM_ALIGNMENT);
            const double alpha = 1, beta = 1;
            init_stack(s, stack_size, 0 /*rnd_size*/, NULL /*rnd*/, mn, mk, kn, nc, na, nb);
#  if defined(_OPENMP)
#    pragma omp parallel
#  endif
            {
#  if defined(_OPENMP)
#    pragma omp for
#  endif
              for (i = 0; i < na; ++i) INIT_MAT(double, i + 42, &a[i * mk], m, k, 1.0 / (nc * na));
#  if defined(_OPENMP)
#    pragma omp for
#  endif
              for (i = 0; i < nb; ++i) INIT_MAT(double, i + 24, &b[i * kn], k, n, 1.0 / (nc * nb));
            }
            memset(c, 0, sizeof(double) * nc * mn);
            start = libxsmm_timer_tick();
            for (i = 0; i < nrepeat; ++i) {
              OPENCL_LIBSMM_GEMM_BATCH(LIBXSMM_DATATYPE_F64, LIBXSMM_DATATYPE_F64, &notrans, &notrans, m, n, k, &alpha, a,
                &m /*lda*/, s + 0 /*stride_a*/, b, &k /*ldb*/, s + 1 /*stride_b*/, &beta, c, &m /*ldc*/, s + 2 /*stride_c*/,
                sizeof(int) * 3, 1 /*index_base*/, stack_size);
            }
            opencl_libsmm_dhst = 1E-9 * ((size_t)2 * m * n * k * stack_size * nrepeat) /
                                 (libxsmm_timer_duration(start, libxsmm_timer_tick()) * OPENCL_LIBSMM_AI(m, n, k, sizeof(double)));
          }
          opencl_libsmm_dacc = (/*sqrt(perfest.gf_ai_dratio_max **/
            exp(perfest.gf_ai_dratio_sumlog / perfest.dcount));
        }
        libxsmm_free(scratch);
      }
    }
  }
  ACC_OPENCL_RETURN(result);
}


int libsmm_acc_finalize(void) {
  /* Routine libsmm_acc_init is called in master thread inside of parallel region
   * However, libsmm_acc_finalize is indirectly called (c_dbcsr_acc_finalize)
   * inside of a parallel region (not just the master thread).
   */
#  if defined(_OPENMP)
  /* initialization/finalization is not meant to be thread-safe */
  int result = ((0 == omp_in_parallel() || /*main*/ 0 == omp_get_thread_num()) ? EXIT_SUCCESS : EXIT_FAILURE);
#  else
  int result = EXIT_SUCCESS;
#  endif
  /* multiple calls to libsmm_acc_finalize are not considered as an error */
  if (0 == LIBXSMM_ATOMIC_SUB_FETCH(&opencl_libsmm_initialized, 1, LIBXSMM_ATOMIC_RELAXED)) {
#  if LIBXSMM_VERSION4(1, 17, 0, 0) < LIBXSMM_VERSION_NUMBER
    char fname[ACC_OPENCL_MAXSTRLEN];
    const void* regentry = libxsmm_get_registry_begin(LIBXSMM_KERNEL_KIND_USER, NULL /*key*/);
    for (; NULL != regentry; regentry = libxsmm_get_registry_next(regentry, NULL /*key*/)) {
      /* opencl_libsmm_trans_t/opencl_libsmm_smm_t carry cl_kernel as 1st data member */
      cl_kernel kernel = *(const cl_kernel*)regentry;
      if (NULL == kernel) kernel = ((const opencl_libsmm_smm_t*)regentry)->kernel[1];
      if (NULL != kernel) { /* only consider user-entry if clGetKernelInfo succeeded */
        cl_int result_entry = clGetKernelInfo(kernel, CL_KERNEL_FUNCTION_NAME, sizeof(fname), fname, NULL);
        if (CL_SUCCESS == result_entry) {
          if (NULL != strstr(fname, OPENCL_LIBSMM_KERNELNAME_TRANS)) { /* trans-kernel */
            result_entry = clReleaseKernel(kernel);
          }
          else if (NULL != strstr(fname, OPENCL_LIBSMM_KERNELNAME_SMM)) { /* SMM-kernel */
            result_entry = clReleaseKernel(kernel);
            if (CL_SUCCESS == result_entry && kernel != ((const opencl_libsmm_smm_t*)regentry)->kernel[1]) {
              kernel = ((const opencl_libsmm_smm_t*)regentry)->kernel[1]; /* release 2nd kernel */
              if (NULL != kernel) result_entry = clReleaseKernel(kernel);
            }
          }
          if (CL_SUCCESS != result_entry) result = result_entry;
        }
      }
    }
#  endif
    opencl_libsmm_shst = opencl_libsmm_dhst = opencl_libsmm_sacc = opencl_libsmm_dacc = 0;
#  if !defined(__DBCSR_ACC)
    /* DBCSR shall call c_dbcsr_acc_init as well as libsmm_acc_init (since both interfaces are used).
     * Also, libsmm_acc_init may privately call c_dbcsr_acc_init (as it depends on the ACC interface).
     * The implementation of c_dbcsr_acc_init should hence be safe against "over initialization".
     * However, DBCSR only calls c_dbcsr_acc_init (and expects an implicit libsmm_acc_init).
     */
    if (EXIT_SUCCESS == result) result = c_dbcsr_acc_finalize();
#  endif
    libxsmm_finalize();
  }
  /* c_dbcsr_acc_finalize is not called since it can be used independently */
  return result;
}


c_dbcsr_acc_bool_t libsmm_acc_is_thread_safe(void) {
  /* match DBCSR's threading level */
#  if defined(_OPENMP)
  return 1;
#  else
  return 0;
#  endif
}


int libsmm_acc_transpose(const int* dev_trs_stack, int offset, int stack_size, void* dev_data, libsmm_acc_data_t datatype, int m,
  int n, int max_kernel_dim, void* stream) {
  int result = EXIT_SUCCESS;
#  if !defined(OPENCL_LIBSMM_SOURCE_TRANSPOSE)
  result = EXIT_FAILURE;
#  else
  const int mn = m * n;
  assert((NULL != dev_trs_stack && NULL != stream && NULL != dev_data && NULL != *ACC_OPENCL_MEM(dev_data) && 0 <= offset &&
           0 <= stack_size) ||
         0 == stack_size);
  if ((
#    if defined(OPENCL_LIBSMM_F64)
        dbcsr_type_real_8 == datatype
#    else
        0
#    endif
        ||
#    if defined(OPENCL_LIBSMM_F32)
        dbcsr_type_real_4 == datatype
#    else
        0
#    endif
        ) &&
      0 < stack_size && 1 < mn && m <= max_kernel_dim && n <= max_kernel_dim)
  {
    const cl_command_queue queue = *ACC_OPENCL_STREAM(stream);
    opencl_libsmm_trans_t* config;
    opencl_libsmm_transkey_t key;
#    if !defined(OPENCL_LIBSMM_VALIDATE_TRANS)
    double duration;
    const libxsmm_timer_tickint start = libxsmm_timer_tick();
#    endif
    LIBXSMM_MEMZERO127(&key); /* potentially heterogeneous key-data (alignment gaps) */
    key.type = datatype;
    key.m = m;
    key.n = n; /* initialize key */
    config = (opencl_libsmm_trans_t*)libxsmm_xdispatch(&key, sizeof(key));
    if (NULL == config) {
      char buffer[ACC_OPENCL_BUFFERSIZE], build_params[ACC_OPENCL_BUFFERSIZE];
      char fname[ACC_OPENCL_MAXSTRLEN];
      int nchar = LIBXSMM_SNPRINTF(fname, sizeof(fname),
        /* kernel name are meant to be unambiguous (BLAS-typeprefix and kernelsize) */
        "x" OPENCL_LIBSMM_KERNELNAME_TRANS "%ix%i", m, n);
#    if defined(__DBCSR_ACC)
      int routine_handle;
      c_dbcsr_timeset(LIBSMM_ACC_TRANSPOSE_ROUTINE_NAME_STRPTR, LIBSMM_ACC_TRANSPOSE_ROUTINE_NAME_LENPTR, &routine_handle);
#    endif
      if (0 < nchar && (int)sizeof(fname) > nchar) {
        cl_device_id active_device;
        result = clGetCommandQueueInfo(queue, CL_QUEUE_DEVICE, sizeof(cl_device_id), &active_device, NULL);
        if (CL_SUCCESS == result) {
          const char* const param_format = "-DGLOBAL=%s -DINPLACE=%i -DFN=%s -DSM=%i -DSN=%i -DSWG=%i -DT=%s";
          const char* const cmem = (EXIT_SUCCESS != opencl_libsmm_use_cmem(active_device) ? "global" : "constant");
          const char *const env_options = getenv("OPENCL_LIBSMM_TRANS_BUILDOPTS"), *tname = "";
          const char* const env_inplace = getenv("OPENCL_LIBSMM_TRANS_INPLACE");
          const char* const env_bm = getenv("OPENCL_LIBSMM_TRANS_BM");
#    if defined(OPENCL_LIBSMM_TRANS_INPLACE)
          const int inplace = ((m == n) && (NULL == env_inplace ? 1 : ('0' != *env_inplace)));
#    else
          const int inplace = ((m == n) && (NULL == env_inplace ? 0 : ('0' != *env_inplace)));
#    endif
          const int blockm = ((NULL == env_bm || '\0' == *env_bm) ? 0 : atoi(env_bm));
          const int bm = (0 >= blockm ? (NULL == config ? /*default*/ m : /*LIBXSMM_CLMP(config->bm, 1, m)*/ m)
                                      : LIBXSMM_MIN(blockm, m));
          size_t wgsize_max;
          opencl_libsmm_trans_t new_config;
          memset(&new_config, 0, sizeof(new_config));
          result = c_dbcsr_acc_opencl_wgsize(active_device, NULL /*kernel*/, &wgsize_max, NULL /*prefmult*/);
          if (EXIT_SUCCESS == result) {
            switch (datatype) {
              case dbcsr_type_real_8: {
                tname = "char8"; /* double */
                fname[0] = 'd';
              } break;
              case dbcsr_type_real_4: {
                tname = "float";
                fname[0] = 's';
              } break;
              default: assert('\0' == *tname);
            }
            new_config.wgsize = LIBXSMM_MIN((size_t)((m == bm || 0 == (m % bm)) ? bm : m), wgsize_max);
            nchar = LIBXSMM_SNPRINTF(buffer, sizeof(buffer), "%s", NULL == env_options ? "" : env_options);
            if (0 <= /*<*/ nchar && (int)sizeof(buffer) > nchar) {
              nchar = LIBXSMM_SNPRINTF(
                build_params, sizeof(build_params), param_format, cmem, inplace, fname, m, n, (int)new_config.wgsize, tname);
            }
          }
          if ('\0' != *tname && 0 < nchar && (int)sizeof(build_params) > nchar) {
            result = c_dbcsr_acc_opencl_kernel(0 /*source_is_file*/, OPENCL_LIBSMM_SOURCE_TRANSPOSE, fname, build_params, buffer,
              NULL /*try*/, NULL /*try_ok*/, NULL /*extnames*/, 0 /*num_exts*/, &new_config.kernel);
            if (EXIT_SUCCESS == result) {
              result = c_dbcsr_acc_opencl_wgsize(active_device, new_config.kernel, &wgsize_max, NULL /*prefmult*/);
              if (EXIT_SUCCESS == result) {
                assert(0 < wgsize_max);
                if (wgsize_max < new_config.wgsize) {
                  new_config.wgsize = wgsize_max;
                  nchar = LIBXSMM_SNPRINTF(
                    build_params, sizeof(build_params), param_format, cmem, inplace, fname, m, n, (int)new_config.wgsize, tname);
                  if (0 < nchar && (int)sizeof(build_params) > nchar) {
                    result = c_dbcsr_acc_opencl_kernel(0 /*source_is_file*/, OPENCL_LIBSMM_SOURCE_TRANSPOSE, fname, build_params,
                      buffer, NULL /*try*/, NULL /*try_ok*/, NULL /*extnames*/, 0 /*num_exts*/, &new_config.kernel);
                  }
                  else result = EXIT_FAILURE;
                }
                if (EXIT_SUCCESS == result) {
                  config = (opencl_libsmm_trans_t*)libxsmm_xregister(&key, sizeof(key), sizeof(new_config), &new_config);
#    if !defined(OPENCL_LIBSMM_VALIDATE_TRANS)
                  if (2 <= c_dbcsr_acc_opencl_config.verbosity || 0 > c_dbcsr_acc_opencl_config.verbosity) {
                    LIBXSMM_STDIO_ACQUIRE();
                    duration = libxsmm_timer_duration(start, libxsmm_timer_tick());
                    fprintf(stderr, "INFO ACC/OpenCL: TRANS-kernel ");
                    opencl_libsmm_write_trans_params(
                      stderr, 0 /*only_key*/, &key, NULL /*config*/, NULL /*delim*/, NULL /*begin*/, NULL /*close*/);
                    fprintf(stderr, "=");
                    opencl_libsmm_write_trans_params(
                      stderr, 0 /*only_key*/, &key, config, NULL /*delim*/, NULL /*begin*/, NULL /*close*/);
                    fprintf(stderr, " gen=%.1f ms\n", 1E3 * duration);
                    LIBXSMM_STDIO_RELEASE();
                  }
#    endif
                }
              }
            }
          }
          else if (EXIT_SUCCESS == result) {
            result = EXIT_FAILURE;
          }
        }
      }
      else {
        result = EXIT_FAILURE;
      }
#    if defined(__DBCSR_ACC)
      c_dbcsr_timestop(&routine_handle);
#    endif
    }
    assert((NULL != config && NULL != config->kernel && 0 < config->wgsize) || EXIT_SUCCESS != result);
    if (EXIT_SUCCESS == result) {
      cl_event event, *const perf_event = ((c_dbcsr_acc_opencl_timer_host == c_dbcsr_acc_opencl_config.timer ||
                                             (0 <= c_dbcsr_acc_opencl_config.verbosity && 2 >= c_dbcsr_acc_opencl_config.verbosity))
                                             ? NULL
                                             : &event);
      const size_t work_size = config->wgsize * stack_size;
      const int typesize = OPENCL_LIBSMM_TYPESIZE(datatype);
#    if defined(OPENCL_LIBSMM_VALIDATE_TRANS)
      const int offset_stack_size = offset + stack_size;
      char *imat = NULL, *omat = NULL, *gold = NULL;
      void* scratch = NULL;
      int* stack = NULL;
      size_t data_size;
      if (CL_SUCCESS == clGetMemObjectInfo(*ACC_OPENCL_MEM(dev_data), CL_MEM_SIZE, sizeof(size_t), &data_size, NULL)) {
        const size_t scratch_size = (sizeof(int) * offset_stack_size) /*stack*/
                                    + data_size /*imat*/ + data_size /*omat*/ + (mn * typesize) /*gold*/
                                    + 3 * (LIBXSMM_ALIGNMENT - 1) /*alignments*/;
        scratch = libxsmm_aligned_scratch(scratch_size, LIBXSMM_ALIGNMENT);
        if (NULL != scratch) {
          stack = (int*)scratch;
          imat = (char*)LIBXSMM_UP2((uintptr_t)stack + sizeof(int) * offset_stack_size, LIBXSMM_ALIGNMENT);
          omat = (char*)LIBXSMM_UP2((uintptr_t)imat + data_size, LIBXSMM_ALIGNMENT);
          gold = (char*)LIBXSMM_UP2((uintptr_t)omat + data_size, LIBXSMM_ALIGNMENT);
          ACC_OPENCL_CHECK(
            c_dbcsr_acc_memcpy_d2h(dev_trs_stack, stack, sizeof(int) * offset_stack_size, stream), "transfer debug stack", result);
          ACC_OPENCL_CHECK(c_dbcsr_acc_memcpy_d2h(dev_data, imat, data_size, stream), "transfer debug input", result);
        }
        else result = EXIT_FAILURE;
      }
      else {
        result = EXIT_FAILURE;
      }
#    endif
      assert(!(OPENCL_LIBSMM_NLOCKS_TRANS & (OPENCL_LIBSMM_NLOCKS_TRANS - 1))); /* POT */
      { /* OpenCL is thread-safe except for clSetKernelArg and launching such shared kernel */
        static volatile int locks[OPENCL_LIBSMM_NLOCKS_TRANS];
#    if (1 < OPENCL_LIBSMM_NLOCKS_TRANS)
        const unsigned int hash = libxsmm_hash(&config->kernel, sizeof(cl_kernel), 25071975 /*seed*/);
        const unsigned int lidx = LIBXSMM_MOD2(hash, OPENCL_LIBSMM_NLOCKS_TRANS);
        volatile int* const lock = locks + lidx;
#    else
        volatile int* const lock = locks;
#    endif
        /* calling clSetKernelArg must be consistent across host-threads */
        LIBXSMM_ATOMIC_ACQUIRE(lock, LIBXSMM_SYNC_NPAUSE, LIBXSMM_ATOMIC_RELAXED);
        ACC_OPENCL_CHECK(clSetKernelArg(config->kernel, 0, sizeof(cl_mem), ACC_OPENCL_MEM(dev_trs_stack)),
          "set batch-list argument of transpose kernel", result);
        ACC_OPENCL_CHECK(
          clSetKernelArg(config->kernel, 1, sizeof(int), &offset), "set offset argument of transpose kernel", result);
        ACC_OPENCL_CHECK(clSetKernelArg(config->kernel, 2, sizeof(cl_mem), ACC_OPENCL_MEM(dev_data)),
          "set matrix-data argument of transpose kernel", result);
        ACC_OPENCL_CHECK(clEnqueueNDRangeKernel(queue, config->kernel, 1 /*work_dim*/, NULL /*offset*/, &work_size, &config->wgsize,
                           0, NULL, perf_event),
          "launch transpose kernel", result);
        /* eventually update performance counters inside of locked region */
#    if !defined(OPENCL_LIBSMM_VALIDATE_TRANS)
        if (3 <= c_dbcsr_acc_opencl_config.verbosity || 0 > c_dbcsr_acc_opencl_config.verbosity) {
          if (NULL != perf_event) {
            cl_ulong begin = 0, end = 0;
            clWaitForEvents(1, perf_event);
            ACC_OPENCL_CHECK(clGetEventProfilingInfo(*perf_event, CL_PROFILING_COMMAND_START, sizeof(cl_ulong), &begin, NULL),
              "query kernel start time", result);
            ACC_OPENCL_CHECK(clGetEventProfilingInfo(*perf_event, CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &end, NULL),
              "query kernel end time", result);
            duration = 1E-9 * LIBXSMM_DELTA(begin, end); /* Nanoseconds->seconds */
          }
          else {
            clFinish(queue);
            duration = libxsmm_timer_duration(start, libxsmm_timer_tick()); /* seconds */
          }
          if (EXIT_SUCCESS == result) {
            const double membw = (1ULL * stack_size * (typesize * m * n)) / (duration * (1ULL << 30));
            const int* const priority = c_dbcsr_acc_opencl_stream_priority(stream);
            LIBXSMM_STDIO_ACQUIRE();
            fprintf(stderr, "INFO ACC/OpenCL: TRANS-kernel ");
            opencl_libsmm_write_trans_params(
              stderr, 1 /*only_key*/, &key, NULL /*config*/, NULL /*delim*/, NULL /*begin*/, NULL /*close*/);
            fprintf(stderr, "=");
            opencl_libsmm_write_trans_params(stderr, 1 /*only_key*/, &key, config, NULL /*delim*/, NULL /*begin*/, NULL /*close*/);
            fprintf(stderr, " prio=%i ss=%i cur=%.1f GB/s dur=%.2g ms\n", NULL != priority ? *priority : -1, stack_size, membw,
              1E3 * duration);
            LIBXSMM_STDIO_RELEASE();
          }
        }
#    endif
        LIBXSMM_ATOMIC_RELEASE(lock, LIBXSMM_ATOMIC_RELAXED);
      }
#    if defined(OPENCL_LIBSMM_VALIDATE_TRANS)
      ACC_OPENCL_CHECK(c_dbcsr_acc_memcpy_d2h(dev_data, omat, data_size, stream), "transfer debug test", result);
#    endif
#    if defined(OPENCL_LIBSMM_VALIDATE_TRANS)
      ACC_OPENCL_CHECK(c_dbcsr_acc_stream_sync(stream), "sync stream", result);
#    endif
#    if defined(OPENCL_LIBSMM_VALIDATE_TRANS)
      if (EXIT_SUCCESS == result) {
        int i, j;
        LIBXSMM_STDIO_ACQUIRE();
        if (0 != c_dbcsr_acc_opencl_config.verbosity) {
          fprintf(stderr,
            "libsmm_acc_transpose("
            "offset=%i, size=%i, type=%s, m=%i, n=%i, max=%i, stream=%p)",
            offset, stack_size, dbcsr_type_real_8 == datatype ? "f64" : (dbcsr_type_real_4 == datatype ? "f32" : "unknown"), m, n,
            max_kernel_dim, stream);
        }
        for (i = offset; i < offset_stack_size; ++i) {
          const size_t index = stack[i];
          const char* const orig = imat + index * typesize;
          const char* const test = omat + index * typesize;
          assert((index * typesize) < data_size);
          memcpy(gold, orig, mn * typesize);
          libxsmm_itrans(gold, typesize, m, n, m, n);
          if (0 != memcmp(gold, test, mn * typesize)) {
            if (0 == c_dbcsr_acc_opencl_config.verbosity) {
              fprintf(stderr,
                "libsmm_acc_transpose("
                "offset=%i, size=%i, type=%s, m=%i, n=%i, max=%i, stream=%p)",
                offset, stack_size, dbcsr_type_real_8 == datatype ? "f64" : (dbcsr_type_real_4 == datatype ? "f32" : "unknown"), m,
                n, max_kernel_dim, stream);
            }
            fprintf(stderr, " => ERROR\n");
#      if defined(_DEBUG)
            opencl_libsmm_print_matrix(stderr, "orig = ", datatype, orig, m, n);
            opencl_libsmm_print_matrix(stderr, "gold = ", datatype, gold, n, m);
            opencl_libsmm_print_matrix(stderr, "test = ", datatype, test, n, m);
            fprintf(stderr, "\n");
#      endif
#      if defined(OPENCL_LIBSMM_VALIDATE_EXIT)
            exit(EXIT_FAILURE);
#      else
            result = EXIT_FAILURE;
            break;
#      endif
          }
          for (j = offset; j < i; ++j) {
            const size_t duplicate = stack[j];
            if (index == duplicate) {
              fprintf(stderr, " => ERROR\n");
#      if defined(OPENCL_LIBSMM_VALIDATE_EXIT)
              exit(EXIT_FAILURE);
#      else
              i = offset_stack_size;
              result = EXIT_FAILURE;
              break;
#      endif
            }
          }
        }
        if (0 != c_dbcsr_acc_opencl_config.verbosity && EXIT_SUCCESS == result) {
          fprintf(stderr, " => OK\n");
        }
        LIBXSMM_STDIO_RELEASE();
      }
      libxsmm_free(scratch);
#    endif
    }
  }
#  endif
  ACC_OPENCL_RETURN(result);
}


c_dbcsr_acc_bool_t libsmm_acc_process_suitable(
  c_dbcsr_acc_bool_t def_mnk, libsmm_acc_data_t datatype, int stack_size, int m_max, int n_max, int k_max, int max_kernel_dim) {
  c_dbcsr_acc_bool_t result = 0; /* false */
  double hst = 0, acc = 0;
  if (0 < m_max && 0 < n_max && 0 < k_max && 0 < stack_size &&
      0 != def_mnk /*homogeneous*/
      /* allow k_max to exceed max_kernel_dim, TODO: BLAS for large kernels (m,n) */
      && m_max <= max_kernel_dim && n_max <= max_kernel_dim)
  {
    switch (datatype) {
#  if defined(OPENCL_LIBSMM_F64)
      case dbcsr_type_real_8: {
        hst = opencl_libsmm_dhst;
        acc = opencl_libsmm_dacc;
        if (0 >= hst || 0 >= acc || hst < acc) result = 1; /* true */
      } break;
#  endif
#  if defined(OPENCL_LIBSMM_F32)
      case dbcsr_type_real_4: {
        hst = opencl_libsmm_shst;
        acc = opencl_libsmm_sacc;
        if (0 >= hst || 0 >= acc || hst < acc) result = 1; /* true */
      } break;
#  endif
      default: assert(/*false*/ 0 == result);
    }
  }
  if ((/*false*/ 0 == result) && (2 <= c_dbcsr_acc_opencl_config.verbosity || 0 > c_dbcsr_acc_opencl_config.verbosity)) {
    opencl_libsmm_smmkey_t key;
    opencl_libsmm_smm_t dummy;
    key.type = datatype;
    key.m = m_max;
    key.n = n_max;
    key.k = k_max; /* initialize key */
    memset(&dummy, 0, sizeof(dummy)); /* mute warnings about potentially uninitialized data */
    LIBXSMM_STDIO_ACQUIRE();
    fprintf(stderr, "INFO ACC/OpenCL: SMM-kernel ");
    opencl_libsmm_write_smm_params(stderr, 1 /*only_key*/, &key, NULL /*config*/, NULL /*delim*/, NULL /*begin*/, NULL /*close*/);
    fprintf(stderr, "=");
    opencl_libsmm_write_smm_params(stderr, 1 /*only_key*/, &key, &dummy, NULL /*delim*/, NULL /*begin*/, NULL /*close*/);
    fprintf(stderr, " ss=%i", stack_size);
    if (m_max <= max_kernel_dim && n_max <= max_kernel_dim) {
      if (0 < hst && 0 < acc) {
        const double ai = OPENCL_LIBSMM_AI(m_max, n_max, k_max, OPENCL_LIBSMM_TYPESIZE(datatype));
        fprintf(stderr, " hst=%.1f acc=%.1f GFLOPS/s is not suitable\n", ai * hst, ai * acc);
      }
      else {
        fprintf(stderr, 0 != def_mnk ? " is ignored\n" : " is inhomogeneous\n");
      }
    }
    else fprintf(stderr, " is too large\n");
    LIBXSMM_STDIO_RELEASE();
  }
  return result;
}


int libsmm_acc_process(const int* host_param_stack, const int* dev_param_stack, int stack_size, libsmm_acc_data_t datatype,
  const void* dev_a_data, const void* dev_b_data, void* dev_c_data, int m_max, int n_max, int k_max, int max_kernel_dim,
  c_dbcsr_acc_bool_t def_mnk, void* stream, void* c_stream) {
  int result = EXIT_SUCCESS;
  const int nparams = 3;
#  if !defined(OPENCL_LIBSMM_SOURCE_MULTIPLY)
  result = EXIT_FAILURE;
#  else
  LIBXSMM_UNUSED(c_stream); /* TODO */
  assert(0 == stack_size || (NULL != dev_a_data && NULL != *ACC_OPENCL_MEM(dev_a_data)));
  assert(0 == stack_size || (NULL != dev_b_data && NULL != *ACC_OPENCL_MEM(dev_b_data)));
  assert(0 == stack_size || (NULL != dev_c_data && NULL != *ACC_OPENCL_MEM(dev_c_data)));
  assert(0 == stack_size || (NULL != host_param_stack && NULL != dev_param_stack));
  assert(0 < nparams && 0 < max_kernel_dim && NULL != stream);
  assert(0 <= stack_size && 0 <= m_max && 0 <= n_max && 0 <= k_max);
  if (0 != libsmm_acc_process_suitable(def_mnk, datatype, stack_size, m_max, n_max, k_max, max_kernel_dim)) {
    opencl_libsmm_smmkey_t key;
#    if !defined(OPENCL_LIBSMM_VALIDATE_SMM)
    double duration;
    const libxsmm_timer_tickint start = libxsmm_timer_tick();
#    endif
    const c_dbcsr_acc_opencl_info_stream_t* const qinfo = c_dbcsr_acc_opencl_info_stream(stream);
    const c_dbcsr_acc_opencl_device_t* const devinfo = c_dbcsr_acc_opencl_config.device + qinfo->tid;
    const cl_command_queue queue = *ACC_OPENCL_STREAM(stream);
    LIBXSMM_MEMZERO127(&key); /* potentially heterogeneous key-data */
    key.devuid = ((1 != c_dbcsr_acc_opencl_config.devmatch && ((unsigned int)-1) != c_dbcsr_acc_opencl_config.devmatch)
                    ? c_dbcsr_acc_opencl_config.devmatch
                    : devinfo->uid);
    key.type = datatype;
    key.m = m_max;
    key.n = n_max;
    key.k = k_max;
    if (CL_SUCCESS == result) {
      static volatile int locks[OPENCL_LIBSMM_NLOCKS_SMM]; /* OpenCL is thread-safe except for clSetKernelArg */
      const char *const env_s = getenv("OPENCL_LIBSMM_SMM_S"), *const env_bs = getenv("OPENCL_LIBSMM_SMM_BS");
      const int s = ((NULL == env_s || '\0' == *env_s) ? OPENCL_LIBSMM_SMM_S : atoi(env_s));
      int kernel_idx = 0, bs = ((NULL == env_bs || '\0' == *env_bs) ? 0 : atoi(env_bs));
      opencl_libsmm_smm_t* config;
      volatile int* lock = locks;
#    if (1 < OPENCL_LIBSMM_NLOCKS_SMM)
      assert(!(OPENCL_LIBSMM_NLOCKS_SMM & (OPENCL_LIBSMM_NLOCKS_SMM - 1))); /* POT */
      lock += LIBXSMM_MOD2(libxsmm_hash(&key, sizeof(key), 25071975 /*seed*/), OPENCL_LIBSMM_NLOCKS_SMM);
#    endif
      LIBXSMM_ATOMIC_ACQUIRE(lock, LIBXSMM_SYNC_NPAUSE, LIBXSMM_ATOMIC_RELAXED);
      config = (opencl_libsmm_smm_t*)libxsmm_xdispatch(&key, sizeof(key));
      if (0 >= bs) bs = ((NULL != config && 0 < config->bs) ? config->bs : OPENCL_LIBSMM_DEFAULT_BS);
      /* determine kernel-kind (mini-batch vs. mini-kernel) */
      if (1 == bs || 0 > s || (bs * s) > stack_size) kernel_idx = bs = 1;
      if (NULL == config || NULL == config->kernel[kernel_idx]) {
        char buffer[ACC_OPENCL_BUFFERSIZE], build_params[ACC_OPENCL_BUFFERSIZE];
        char fname[ACC_OPENCL_MAXSTRLEN];
        int cl_level_major, nchar = LIBXSMM_SNPRINTF(fname, sizeof(fname),
                              /* kernel name are meant to be unambiguous (BLAS-typeprefix and kernelsize) */
                              "x" OPENCL_LIBSMM_KERNELNAME_SMM "%ix%ix%i", m_max, n_max, k_max);
        const char* extensions[] = {NULL, NULL};
        cl_device_id active_device = NULL;
        cl_device_type device_type = 0;
#    if defined(__DBCSR_ACC)
        int routine_handle;
        c_dbcsr_timeset(LIBSMM_ACC_PROCESS_ROUTINE_NAME_STRPTR, LIBSMM_ACC_PROCESS_ROUTINE_NAME_LENPTR, &routine_handle);
#    endif
        result = ((0 < nchar && (int)sizeof(fname) > nchar)
                    ? clGetCommandQueueInfo(queue, CL_QUEUE_DEVICE, sizeof(cl_device_id), &active_device, NULL)
                    : EXIT_FAILURE);
        if (EXIT_SUCCESS == result) {
          result = c_dbcsr_acc_opencl_device_level(
            active_device, &cl_level_major, NULL /*level_minor*/, NULL /*cl_std*/, &device_type);
        }
        if (EXIT_SUCCESS == result) {
          const char *tname = NULL, *atomic_type = "";
          int std_c11 = 0;
          switch (datatype) {
            case dbcsr_type_real_8: {
              extensions[0] = "cl_khr_fp64 cl_khr_int64_base_atomics cl_khr_int64_extended_atomics";
              tname = "double";
              fname[0] = 'd';
              if (2 <= cl_level_major && EXIT_SUCCESS == c_dbcsr_acc_opencl_device_ext(active_device, extensions, 1)) {
                atomic_type = "-DTA=long -DTA2=atomic_long -DTF=atomic_double";
                std_c11 = 1;
              }
              else {
                extensions[0] = "cl_khr_fp64 cl_khr_int64_base_atomics";
                if (EXIT_SUCCESS == c_dbcsr_acc_opencl_device_ext(active_device, extensions, 1)) {
                  atomic_type = "-DTA=long";
                }
                else { /* fallback */
                  extensions[0] = "cl_khr_fp64 cl_khr_global_int32_base_atomics cl_khr_global_int32_extended_atomics";
                  if (2 <= cl_level_major && EXIT_SUCCESS == c_dbcsr_acc_opencl_device_ext(active_device, extensions, 1)) {
                    atomic_type = "-DATOMIC32_ADD64 -DTA=int -DTA2=atomic_int -DTF=atomic_double";
                    std_c11 = 1;
                  }
                  else {
                    extensions[0] = "cl_khr_fp64 cl_khr_global_int32_base_atomics";
                    if (EXIT_SUCCESS == c_dbcsr_acc_opencl_device_ext(active_device, extensions, 1)) {
                      atomic_type = "-DATOMIC32_ADD64 -DTA=int";
                    }
                    else tname = NULL;
                  }
                }
              }
            } break;
            case dbcsr_type_real_4: {
              extensions[0] = "cl_khr_global_int32_base_atomics cl_khr_global_int32_extended_atomics";
              if (2 <= cl_level_major && EXIT_SUCCESS == c_dbcsr_acc_opencl_device_ext(active_device, extensions, 1)) {
                extensions[1] = "cl_khr_int64_base_atomics cl_khr_int64_extended_atomics";
                atomic_type = "-DTA=int -DTA2=atomic_int -DTF=atomic_float";
                std_c11 = 1;
                tname = "float";
                fname[0] = 's';
              }
              else {
                extensions[0] = "cl_khr_global_int32_base_atomics";
                if (EXIT_SUCCESS == c_dbcsr_acc_opencl_device_ext(active_device, extensions, 1)) {
                  extensions[1] = "cl_khr_int64_base_atomics";
                  atomic_type = "-DTA=int";
                  tname = "float";
                  fname[0] = 's';
                }
              }
            } break;
            default: assert(NULL == tname);
          }
          if (NULL != tname) {
            const char* const env_devid = getenv("OPENCL_LIBSMM_SMM_DEVID");
            const unsigned int devuid = (NULL == env_devid || '\0' == *env_devid) ? devinfo->uid
                                                                                  : (unsigned int)strtoul(env_devid, NULL, 0);
            size_t wgsize_max, wgsize_prf, sgs = 0;
            opencl_libsmm_smm_t new_config;
            if (NULL == config) {
              memset(&new_config, 0, sizeof(new_config));
            }
            else { /* preserve kernels, performance counters, etc. */
              memcpy(&new_config, config, sizeof(opencl_libsmm_smm_t));
            }
            result = c_dbcsr_acc_opencl_wgsize(active_device, NULL /*device-specific*/, &wgsize_max, &wgsize_prf);
            assert(EXIT_SUCCESS != result || 0 < wgsize_prf);
            if (EXIT_SUCCESS == result) {
              const char *const env_bm = getenv("OPENCL_LIBSMM_SMM_BM"), *const env_bn = getenv("OPENCL_LIBSMM_SMM_BN");
              const char *const env_bk = getenv("OPENCL_LIBSMM_SMM_BK"), *const env_ws = getenv("OPENCL_LIBSMM_SMM_WS");
              const char *const env_wg = getenv("OPENCL_LIBSMM_SMM_WG"), *const env_lu = getenv("OPENCL_LIBSMM_SMM_LU");
              const char *const env_nz = getenv("OPENCL_LIBSMM_SMM_NZ"), *const env_al = getenv("OPENCL_LIBSMM_SMM_AL");
              const char *const env_tb = getenv("OPENCL_LIBSMM_SMM_TB"), *const env_tc = getenv("OPENCL_LIBSMM_SMM_TC");
              const char *const env_ap = getenv("OPENCL_LIBSMM_SMM_AP"), *const env_aa = getenv("OPENCL_LIBSMM_SMM_AA");
              const char *const env_ab = getenv("OPENCL_LIBSMM_SMM_AB"), *const env_ac = getenv("OPENCL_LIBSMM_SMM_AC");
              const int blockm = ((NULL == env_bm || '\0' == *env_bm) ? 0 : atoi(env_bm));
              const int blockn = ((NULL == env_bn || '\0' == *env_bn) ? 0 : atoi(env_bn));
              const int blockk = ((NULL == env_bk || '\0' == *env_bk) ? 0 : atoi(env_bk));
              const int wgmin = ((NULL == env_ws || '\0' == *env_ws) ? 0 : atoi(env_ws));
              const int default_aa = (((0x0bd0 > devuid || 0x0bdb < devuid)) ? ((k_max % OPENCL_LIBSMM_VMIN) ? 1 : 2) : 0);
              const int default_ab = (((0x0bd0 > devuid || 0x0bdb < devuid) && 0x020a != devuid) ? 3 : 0), default_ac = 0;
              const int default_bk = (((0x0bd0 > devuid || 0x0bdb < devuid) && 0x020a != devuid)
                                        ? (0 == kernel_idx ? LIBXSMM_MIN(OPENCL_LIBSMM_DEFAULT_BK, m_max)
                                                           : LIBXSMM_MIN(OPENCL_LIBSMM_VMIN, m_max))
                                        : 1);
              const int default_wg = (((0x0bd0 > devuid || 0x0bdb < devuid)) ? (0 == kernel_idx ? 0 : -2) : -1);
              const int default_lu = (0 != devinfo->intel ? -1 : 0);
              int nbm, nbn;
              /* two defaults for new_config parameters: 1st - regular, 2nd - BS=1 kernel */
              new_config.bm = (0 >= blockm ? (0 == kernel_idx ? (NULL == config ? LIBXSMM_MIN(OPENCL_LIBSMM_DEFAULT_BM, m_max)
                                                                                : LIBXSMM_CLMP(config->bm, 1, m_max))
                                                              : LIBXSMM_MIN(OPENCL_LIBSMM_DEFAULT_BM, m_max))
                                           : LIBXSMM_MIN(blockm, m_max));
              new_config.bn = (0 >= blockn ? (0 == kernel_idx ? (NULL == config ? LIBXSMM_MIN(OPENCL_LIBSMM_DEFAULT_BN, n_max)
                                                                                : LIBXSMM_CLMP(config->bn, 1, n_max))
                                                              : LIBXSMM_MIN(OPENCL_LIBSMM_DEFAULT_BN, n_max))
                                           : LIBXSMM_MIN(blockn, n_max));
              new_config.bk = (0 >= blockk ? (NULL == config ? default_bk : LIBXSMM_CLMP(config->bk, 1, m_max))
                                           : LIBXSMM_MIN(blockk, m_max));
              new_config.ws = (0 >= wgmin ? (0 == kernel_idx ? (NULL == config ? /*default*/ LIBXSMM_MAX(m_max, n_max)
                                                                               : LIBXSMM_CLMP(config->ws, 1, n_max * m_max))
                                                             : /*default*/ LIBXSMM_MAX(m_max, n_max))
                                          : LIBXSMM_MIN(wgmin, n_max * m_max));
              new_config.wg = LIBXSMM_CLMP(
                (NULL == env_wg || '\0' == *env_wg) ? (NULL == config ? default_wg : config->wg) : atoi(env_wg), -2, 2);
              new_config.lu = LIBXSMM_MAX(-2, (NULL == env_lu || '\0' == *env_lu)
                                                ? (0 == kernel_idx ? (NULL == config ? default_lu : config->lu) : default_lu)
                                                : atoi(env_lu)); /* populate only lower bound */
              new_config.nz = LIBXSMM_CLMP((NULL == env_nz || '\0' == *env_nz)
                                             ? (0 == kernel_idx ? (NULL == config ? /*default*/ 0 : config->nz) : /*default*/ 0)
                                             : atoi(env_nz),
                0, 1);
              new_config.al = LIBXSMM_CLMP((NULL == env_al || '\0' == *env_al)
                                             ? (0 == kernel_idx ? (NULL == config ? /*default*/ 0 : config->al) : /*default*/ 0)
                                             : atoi(env_al),
                0, 1);
              new_config.tb = LIBXSMM_CLMP((NULL == env_tb || '\0' == *env_tb)
                                             ? (0 == kernel_idx ? (NULL == config ? /*default*/ 0 : config->tb) : /*default*/ 0)
                                             : atoi(env_tb),
                0, 1);
              new_config.tc = LIBXSMM_CLMP((NULL == env_tc || '\0' == *env_tc)
                                             ? (0 == kernel_idx ? (NULL == config ? /*default*/ 1 : config->tc) : /*default*/ 1)
                                             : atoi(env_tc),
                0, 1);
              new_config.ap = LIBXSMM_CLMP((NULL == env_ap || '\0' == *env_ap)
                                             ? (0 == kernel_idx ? (NULL == config ? /*default*/ 0 : config->ap) : /*default*/ 0)
                                             : atoi(env_ap),
                0, 1);
              new_config.aa = LIBXSMM_CLMP(
                (NULL == env_aa || '\0' == *env_aa)
                  ? (0 == kernel_idx ? (NULL == config ? /*default*/ default_aa : config->aa) : /*default*/ default_aa)
                  : atoi(env_aa),
                0, 3);
              new_config.ab = LIBXSMM_CLMP(
                (NULL == env_ab || '\0' == *env_ab)
                  ? (0 == kernel_idx ? (NULL == config ? /*default*/ default_ab : config->ab) : /*default*/ default_ab)
                  : atoi(env_ab),
                0, 3);
              new_config.ac = LIBXSMM_CLMP(
                (NULL == env_ac || '\0' == *env_ac)
                  ? (0 == kernel_idx ? (NULL == config ? /*default*/ default_ac : config->ac) : /*default*/ default_ac)
                  : atoi(env_ac),
                0, 2);
              if (0 >= new_config.s) new_config.s = stack_size;
              if (0 == kernel_idx || 1 >= new_config.bs) new_config.bs = bs;
              nbm = (m_max + new_config.bm - 1) / new_config.bm;
              nbn = (n_max + new_config.bn - 1) / new_config.bn;
              new_config.wgsize[kernel_idx] = LIBXSMM_MAX(nbm * nbn, new_config.ws);
#    if LIBXSMM_VERSION4(1, 17, 0, 0) < LIBXSMM_VERSION_NUMBER
              if (0 != new_config.wg) {
                const unsigned int limit = (unsigned int)LIBXSMM_MAX(wgsize_prf, OPENCL_LIBSMM_VLEN);
                unsigned int r = libxsmm_remainder(
                  (unsigned int)new_config.wgsize[kernel_idx], OPENCL_LIBSMM_VMIN, &limit, NULL /*remainder*/);
                if (0 > new_config.wg) {
                  const char* const extension = "cl_intel_required_subgroup_size";
                  if (EXIT_SUCCESS == c_dbcsr_acc_opencl_device_ext(active_device, &extension, 1)) {
                    unsigned int q = limit, i = 0;
                    size_t sizes[16], nbytes = 0;
                    ACC_OPENCL_EXPECT(EXIT_SUCCESS == clGetDeviceInfo(active_device, 0x4108 /*CL_DEVICE_SUB_GROUP_SIZES_INTEL*/,
                                                        sizeof(sizes), sizes, &nbytes));
                    if (-1 == new_config.wg) { /* cover entire WG-size in sub-group size */
                      for (; (i * sizeof(size_t)) < nbytes; ++i) {
                        sgs = sizes[i];
                        if (new_config.wgsize[kernel_idx] <= sgs) break;
                      }
                      if (new_config.wgsize[kernel_idx] > sgs) sgs = 0;
                    }
                    else { /* explicit sub-group size with minimized WG-remainder */
                      for (; (i * sizeof(size_t)) < nbytes; ++i) {
                        r = libxsmm_remainder(
                          (unsigned int)new_config.wgsize[kernel_idx], (unsigned int)sizes[i], &limit, NULL /*remainder*/);
                        if (r <= q) {
                          q = r;
                          sgs = sizes[i];
                        }
                      }
                    }
                    wgsize_prf = new_config.wgsize[kernel_idx];
                  }
                  else wgsize_prf = r;
                }
                else wgsize_prf = r;
              }
              else
#    endif
              {
                wgsize_prf = new_config.wgsize[kernel_idx];
              }
              if (2 <= new_config.wg) wgsize_prf = LIBXSMM_UP2POT(wgsize_prf);
              if (wgsize_prf < (2 * new_config.wgsize[kernel_idx])) new_config.wgsize[kernel_idx] = wgsize_prf; /* limit */
              assert(1 <= bs && 0 < new_config.wgsize[kernel_idx] && 0 < wgsize_max && 0 < wgsize_prf);
              /* ensure minimum requested WG-size */
              while ((nbm * nbn) < new_config.ws && (nbm < m_max || nbn < n_max)) {
                if (nbn < n_max) {
                  ++nbn;
                  new_config.bn = (n_max + nbn - 1) / nbn;
                }
                else if (nbm < m_max) {
                  ++nbm;
                  new_config.bm = (m_max + nbm - 1) / nbm;
                }
                new_config.wgsize[kernel_idx] = (2 > new_config.wg ? (nbm * nbn) : ((int)LIBXSMM_UP2POT(nbm * nbn)));
              }
              /* limit WG-size to maximum WG-size */
              while (wgsize_max < new_config.wgsize[kernel_idx] && (new_config.bm < m_max || new_config.bn < n_max)) {
                if (new_config.bn < n_max) {
                  ++new_config.bn;
                  nbn = (n_max + new_config.bn - 1) / new_config.bn;
                }
                else if (new_config.bm < m_max) {
                  ++new_config.bm;
                  nbm = (m_max + new_config.bm - 1) / new_config.bm;
                }
                new_config.wgsize[kernel_idx] = (2 > new_config.wg ? (nbm * nbn) : ((int)LIBXSMM_UP2POT(nbm * nbn)));
              }
              if (new_config.wgsize[kernel_idx] <= wgsize_max) { /* SMMs can be potentially handled by device */
                const char* const cmem = (EXIT_SUCCESS != opencl_libsmm_use_cmem(active_device) ? "global" : "constant");
                const char* const env_options = getenv("OPENCL_LIBSMM_SMM_BUILDOPTS");
                const char* const env_barrier = getenv("OPENCL_LIBSMM_SMM_BARRIER");
                const char* const env_atomics = getenv("OPENCL_LIBSMM_SMM_ATOMICS");
                const char* const env_nrepeat = getenv("SMM_NREPEAT");
                const char *barrier_expr = NULL, *atomic_ops = "";
                const char *atomic_exp = NULL, *atomic_expr2 = "";
                if (NULL == env_barrier || '0' != *env_barrier) {
                  barrier_expr = ((0 != std_c11 && (0 == devinfo->intel || (CL_DEVICE_TYPE_CPU != device_type)))
                                    ? "-D\"BARRIER(A)=work_group_barrier(A,memory_scope_work_group)\""
                                    : "-D\"BARRIER(A)=barrier(A)\"");
                }
                else barrier_expr = ""; /* no barrier */
                assert(NULL != barrier_expr);
                if (NULL == env_atomics || '0' != *env_atomics) {
                  /* atomics_force: attempt to force atomics without confirmation */
                  const int atomics_force = ((NULL == env_atomics || '\0' == *env_atomics) ? 0 : atoi(env_atomics));
                  const int cl_nonv = (EXIT_SUCCESS != c_dbcsr_acc_opencl_device_vendor(active_device, "nvidia"));
                  if (NULL == env_atomics || '\0' == *env_atomics || 0 != atomics_force) {
                    cl_bitfield fp_atomics;
                    assert(dbcsr_type_real_8 == datatype || dbcsr_type_real_4 == datatype);
                    if (CL_SUCCESS == clGetDeviceInfo(active_device,
                                        (cl_device_info)(dbcsr_type_real_8 == datatype ? 0x4232 : 0x4231), sizeof(cl_bitfield),
                                        &fp_atomics, NULL) &&
                        0 != (/*add*/ (1 << 1) & fp_atomics))
                    {
                      extensions[1] = "cl_ext_float_atomics";
                      atomic_exp = (dbcsr_type_real_8 == datatype
                                      ? "atomic_fetch_add_explicit((GLOBAL_VOLATILE(atomic_double)*)A,B,"
                                        "memory_order_relaxed,memory_scope_work_group)"
                                      : "atomic_fetch_add_explicit((GLOBAL_VOLATILE(atomic_float)*)A,B,"
                                        "memory_order_relaxed,memory_scope_work_group)");
                    }
                    else if ((0 != devinfo->intel && 0x4905 != devinfo->uid && 0 == devinfo->unified) || 0 != atomics_force) {
                      if ((0 != devinfo->intel &&
                            (dbcsr_type_real_4 == datatype || (0x0bd0 <= devinfo->uid && 0x0bdb >= devinfo->uid))) ||
                          (0 != atomics_force))
                      {
                        if (0 == atomics_force && (0 == devinfo->intel || 0x0bd0 > devinfo->uid || 0x0bdb < devinfo->uid)) {
                          extensions[1] = "cl_intel_global_float_atomics";
                          atomic_ops = "-Dcl_intel_global_float_atomics";
                        }
                        else {
                          atomic_ops = ((0 == std_c11 && 2 > atomics_force)
                                          ? "-DATOMIC_PROTOTYPES=1"
                                          : (3 > atomics_force ? "-DATOMIC_PROTOTYPES=2" : "-DATOMIC_PROTOTYPES=3"));
                        }
                        atomic_exp = ((0 == std_c11 && 2 > atomics_force) ? "atomic_add(A,B)"
                                                                          : "atomic_fetch_add_explicit((GLOBAL_VOLATILE(TF)*)A,B,"
                                                                            "memory_order_relaxed,memory_scope_work_group)");
                      }
                      else {
                        atomic_exp = "atomic_add_global_cmpxchg(A,B)";
                        atomic_ops = "-DCMPXCHG=atom_cmpxchg";
                      }
                    }
                    else if (cl_nonv) {
                      if (NULL != extensions[1] && 1 < bs && 1 == new_config.bn && new_config.bm >= m_max && 0 == new_config.al &&
                          (0 == (m_max & 1) || (0 == devinfo->intel /*&& cl_nonv*/)) /* TODO */
                          && EXIT_SUCCESS == c_dbcsr_acc_opencl_device_ext(active_device, extensions + 1, 1))
                      {
                        assert(dbcsr_type_real_4 == datatype);
                        atomic_expr2 = "-D\"ATOMIC_ADD2_GLOBAL(A,B)=atomic_add_global_cmpxchg2(A,B)\"";
                      }
                      else {
                        extensions[1] = NULL;
                      }
                      atomic_exp = "atomic_add_global_cmpxchg(A,B)";
                      atomic_ops = (dbcsr_type_real_4 == datatype ? "-DCMPXCHG=atomic_cmpxchg" : "-DCMPXCHG=atom_cmpxchg");
                    }
                    else {
                      assert(NULL != atomic_ops && '\0' == *atomic_ops);
                      atomic_exp = "atomic_add_global_xchg(A,B)";
                    }
                  }
                  else if (NULL != LIBXSMM_STRISTR(env_atomics, "cmpxchg")) {
                    if (NULL != extensions[1] && 1 < bs && 1 == new_config.bn && new_config.bm >= m_max && 0 == new_config.al &&
                        (0 == (m_max & 1) || (0 == devinfo->intel && cl_nonv)) /* TODO */
                        && '2' == env_atomics[strlen(env_atomics) - 1] &&
                        EXIT_SUCCESS == c_dbcsr_acc_opencl_device_ext(active_device, extensions + 1, 1))
                    {
                      assert(dbcsr_type_real_4 == datatype);
                      atomic_expr2 = "-D\"ATOMIC_ADD2_GLOBAL(A,B)=atomic_add_global_cmpxchg2(A,B)\"";
                    }
                    else {
                      extensions[1] = NULL;
                    }
                    atomic_exp = "atomic_add_global_cmpxchg(A,B)";
                    atomic_ops = (dbcsr_type_real_4 == datatype ? "-DCMPXCHG=atomic_cmpxchg" : "-DCMPXCHG=atom_cmpxchg");
                  }
                  else {
                    atomic_exp = "atomic_add_global_xchg(A,B)";
                    atomic_ops = (dbcsr_type_real_4 == datatype ? "-DXCHG=atomic_xchg" : "-DXCHG=atom_xchg");
                  }
                }
                else { /* unsynchronized */
                  assert(NULL != env_atomics);
                  atomic_exp = "*(A)+=(B)"; /* non-atomic update */
                }
                assert(NULL != atomic_exp);
                /* compose build parameters and flags */
                nchar = LIBXSMM_SNPRINTF(build_params, sizeof(build_params),
                  "-DMAD=fma -DINTEL=%u -DGLOBAL=%s -DSWG=%i -DSGS=%i -DFN=%s -DREPEAT=%i -DLU=%i "
                  "-DSM=%i -DSN=%i -DSK=%i -DBS=%i -DBM=%i -DBN=%i -DBK=%i -DT=%s -DTN=%i "
                  "%s %s %s %s %s %s %s %s %s %s -D\"ATOMIC_ADD_GLOBAL(A,B)=%s\" %s %s",
                  0 != devinfo->intel ? devinfo->uid : 0, cmem, (int)new_config.wgsize[kernel_idx], (int)sgs, fname,
                  NULL == env_nrepeat ? 1 : atoi(env_nrepeat), new_config.lu, m_max, n_max, k_max, bs, new_config.bm, new_config.bn,
                  new_config.bk, tname, datatype, 0 == new_config.nz ? "" : "-DATOMIC_INC_NZ", 0 == new_config.al ? "" : "-DAL",
                  0 == new_config.tb ? "" : "-DTRACK_B", 0 != new_config.tc ? "-DTRACK_C" : "", 0 == new_config.ap ? "" : "-DSLM_P",
                  0 == new_config.aa ? "" : (1 == new_config.aa ? "-DSLM_A=1" : (2 == new_config.aa ? "-DSLM_A=2" : "-DREG_A")),
                  0 == new_config.ab ? "" : (1 == new_config.ab ? "-DSLM_B=1" : (2 == new_config.ab ? "-DSLM_B=2" : "-DREG_B")),
                  0 == new_config.ac ? "" : (1 == new_config.ac ? "-DSLM_C=1" : "-DSLM_C=2"), atomic_type, atomic_ops, atomic_exp,
                  atomic_expr2, barrier_expr);
                if (0 < nchar && (int)sizeof(build_params) > nchar) {
#    if !defined(NDBGDEV)
                  const char* const cl_debug = ((0 != devinfo->intel && CL_DEVICE_TYPE_CPU != device_type) ? "-gline-tables-only"
                                                                                                           : "");
#    else
                  const char* const cl_debug = "";
#    endif
                  nchar = LIBXSMM_SNPRINTF(buffer, sizeof(buffer), "%s %s -cl-fast-relaxed-math -cl-denorms-are-zero",
                    NULL == env_options ? "" : env_options, cl_debug);
                  if (0 >= nchar || (int)sizeof(buffer) <= nchar) result = EXIT_FAILURE;
                }
                else result = EXIT_FAILURE;
              }
              /* matrix-size causes too large WG-size */
              else result = EXIT_FAILURE;
            }
            if (EXIT_SUCCESS == result) {
              const char* const env_kernel = getenv("OPENCL_LIBSMM_SMM_KERNEL");
              result = c_dbcsr_acc_opencl_kernel(NULL == env_kernel ? 0 : 1,
                NULL == env_kernel ? OPENCL_LIBSMM_SOURCE_MULTIPLY : env_kernel, fname, build_params, buffer, NULL /*cl_try*/,
                NULL /*cl_try_ok*/, extensions, sizeof(extensions) / sizeof(*extensions), new_config.kernel + kernel_idx);
              if (EXIT_SUCCESS == result) {
                size_t wgsize_max_kernel = wgsize_max;
                result = c_dbcsr_acc_opencl_wgsize(
                  active_device, new_config.kernel[kernel_idx], &wgsize_max_kernel, NULL /*prefmult*/);
                if (EXIT_SUCCESS == result) {
                  assert(0 < new_config.wgsize[kernel_idx] && 0 < wgsize_max && 0 < wgsize_max_kernel);
                  assert(wgsize_max_kernel <= wgsize_max);
                  if (new_config.wgsize[kernel_idx] <= wgsize_max_kernel) { /* check planned WG-size vs kernel-specific WG-size */
                    if (NULL == config || NULL == config->kernel[kernel_idx]) {
                      config = (opencl_libsmm_smm_t*)libxsmm_xregister(&key, sizeof(key), sizeof(new_config), &new_config);
                    }
                    if (NULL != config) {
#    if !defined(OPENCL_LIBSMM_VALIDATE_SMM)
                      if (2 <= c_dbcsr_acc_opencl_config.verbosity || 0 > c_dbcsr_acc_opencl_config.verbosity) {
                        LIBXSMM_STDIO_ACQUIRE();
                        duration = libxsmm_timer_duration(start, libxsmm_timer_tick());
                        fprintf(stderr, "INFO ACC/OpenCL: SMM-kernel ");
                        opencl_libsmm_write_smm_params(
                          stderr, 0 /*only_key*/, &key, NULL /*config*/, NULL /*delim*/, NULL /*begin*/, NULL /*close*/);
                        fprintf(stderr, "=");
                        opencl_libsmm_write_smm_params(
                          stderr, 0 /*only_key*/, &key, &new_config, NULL /*delim*/, NULL /*begin*/, NULL /*close*/);
                        fprintf(stderr, " gen=%.1f ms\n", 1E3 * duration);
                        LIBXSMM_STDIO_RELEASE();
                      }
#    endif
                    }
                    /* failed to register config */
                    else result = EXIT_FAILURE;
                  }
                  else {
                    if (0 != c_dbcsr_acc_opencl_config.verbosity) {
                      fprintf(stderr, "ERROR LIBSMM: tile-size causes too large WG-size (min(%u,%u) < %u)!\n",
                        (unsigned int)wgsize_max_kernel, (unsigned int)wgsize_max, (unsigned int)new_config.wgsize[kernel_idx]);
                    }
                    result = EXIT_FAILURE; /* tile-size causes too large WG-size */
                  }
                }
              }
            }
          }
          /* insufficient device capabilities */
          else result = EXIT_FAILURE;
        }
        /* remove configuration from registry to avoid infinitely retrying code generation */
        if (EXIT_SUCCESS != result && NULL != config) {
          libxsmm_xrelease(&key, sizeof(key));
        }
#    if defined(__DBCSR_ACC)
        c_dbcsr_timestop(&routine_handle);
#    endif
      }
      assert(EXIT_SUCCESS != result || (NULL != config && NULL != config->kernel[kernel_idx]));
      assert(EXIT_SUCCESS != result || (1 <= config->bm && config->bm <= m_max));
      assert(EXIT_SUCCESS != result || (1 <= config->bn && config->bn <= n_max));
      assert(EXIT_SUCCESS != result || (1 <= config->bk && config->bk <= m_max));
      assert(EXIT_SUCCESS != result || (1 <= config->ws && config->ws <= (m_max * n_max)));
      assert(EXIT_SUCCESS != result || (-2 <= config->wg && 2 >= config->wg));
      assert(EXIT_SUCCESS != result || (-2 <= config->lu /*&& 2 >= config->lu*/));
      assert(EXIT_SUCCESS != result || (0 <= config->nz && 1 >= config->nz));
      assert(EXIT_SUCCESS != result || (0 <= config->al && 1 >= config->al));
      assert(EXIT_SUCCESS != result || (0 <= config->tb && 1 >= config->tb));
      assert(EXIT_SUCCESS != result || (0 <= config->tc && 1 >= config->tc));
      assert(EXIT_SUCCESS != result || (0 <= config->ap && 1 >= config->ap));
      assert(EXIT_SUCCESS != result || (0 <= config->aa && 3 >= config->aa));
      assert(EXIT_SUCCESS != result || (0 <= config->ab && 3 >= config->ab));
      assert(EXIT_SUCCESS != result || (0 <= config->ac && 2 >= config->ac));
      assert(EXIT_SUCCESS != result || (1 <= config->wgsize[kernel_idx]));
      assert(EXIT_SUCCESS != result || (1 <= config->s && 1 <= config->bs));
      if (EXIT_SUCCESS == result) {
        cl_event event, *const perf_event =
                          ((c_dbcsr_acc_opencl_timer_host == c_dbcsr_acc_opencl_config.timer ||
                             (0 <= c_dbcsr_acc_opencl_config.verbosity && 2 >= c_dbcsr_acc_opencl_config.verbosity))
                              ? NULL
                              : &event);
        size_t work_size;
#    if defined(OPENCL_LIBSMM_VALIDATE_SMM)
        /* validate result (implies readback from device and performance penalty) */
        char *ainp = NULL, *binp = NULL, *test = NULL, *gold = NULL, *btrn = NULL;
        const libxsmm_datatype precision =
          (dbcsr_type_real_8 == datatype ? LIBXSMM_DATATYPE_F64
                                         : (dbcsr_type_real_4 == datatype ? LIBXSMM_DATATYPE_F32 : LIBXSMM_DATATYPE_UNSUPPORTED));
        const int typesize = OPENCL_LIBSMM_TYPESIZE(datatype);
        libxsmm_xmmfunction kernel_cpu = {NULL};
        size_t asize, bsize, csize;
        void* scratch = NULL;
        if (CL_SUCCESS == clGetMemObjectInfo(*ACC_OPENCL_MEM(dev_a_data), CL_MEM_SIZE, sizeof(size_t), &asize, NULL) &&
            CL_SUCCESS == clGetMemObjectInfo(*ACC_OPENCL_MEM(dev_b_data), CL_MEM_SIZE, sizeof(size_t), &bsize, NULL) &&
            CL_SUCCESS == clGetMemObjectInfo(*ACC_OPENCL_MEM(dev_c_data), CL_MEM_SIZE, sizeof(size_t), &csize, NULL))
        {
          const double alpha = 1, beta = 1;
          libxsmm_descriptor_blob blob;
          libxsmm_gemm_descriptor* const desc = libxsmm_gemm_descriptor_dinit(
            &blob, precision, m_max, n_max, k_max, m_max, k_max, m_max, alpha, beta, LIBXSMM_GEMM_FLAG_NONE, LIBXSMM_PREFETCH_NONE);
          scratch = libxsmm_aligned_scratch(
            asize + bsize + csize + csize + k_max * n_max * typesize + 4 * (LIBXSMM_ALIGNMENT - 1) /*alignments*/,
            LIBXSMM_ALIGNMENT);
          if (NULL != desc && NULL != scratch) {
            ainp = (char*)scratch;
            binp = (char*)LIBXSMM_UP2((uintptr_t)ainp + asize, LIBXSMM_ALIGNMENT);
            test = (char*)LIBXSMM_UP2((uintptr_t)binp + bsize, LIBXSMM_ALIGNMENT);
            gold = (char*)LIBXSMM_UP2((uintptr_t)test + csize, LIBXSMM_ALIGNMENT);
            btrn = (char*)LIBXSMM_UP2((uintptr_t)gold + csize, LIBXSMM_ALIGNMENT);
            ACC_OPENCL_CHECK(c_dbcsr_acc_memcpy_d2h(dev_a_data, ainp, asize, stream), "transfer debug a-data", result);
            ACC_OPENCL_CHECK(c_dbcsr_acc_memcpy_d2h(dev_b_data, binp, bsize, stream), "transfer debug b-data", result);
            ACC_OPENCL_CHECK(c_dbcsr_acc_memcpy_d2h(dev_c_data, gold, csize, stream), "transfer debug c-data", result);
            kernel_cpu = libxsmm_xmmdispatch(desc);
            assert(NULL != kernel_cpu.xmm);
          }
          else result = EXIT_FAILURE;
        }
        else result = EXIT_FAILURE;
#    endif
        /* scale intra-kernel batchsize according to stacksize */
        if (0 == kernel_idx && 1 < config->bs && stack_size < config->s) {
#    if defined(OPENCL_LIBSMM_BS_MIN)
          const int config_bs = LIBXSMM_MAX(config->bs, OPENCL_LIBSMM_BS_MIN);
#    else
          const int config_bs = config->bs;
#    endif
          bs = (stack_size * config_bs + config->s - 1) / (config->s - 1);
          if (config->bs < bs) bs = config->bs;
        }
        /* adjust launchsize according to intra-kernel batchsize */
        work_size = ((stack_size + bs - 1) / bs) * config->wgsize[kernel_idx];
        /* calling clSetKernelArg must be consistent across host-threads */
        ACC_OPENCL_CHECK(clSetKernelArg(config->kernel[kernel_idx], 0, sizeof(cl_mem), ACC_OPENCL_MEM(dev_c_data)),
          "set C-matrix argument of SMM-kernel", result);
        ACC_OPENCL_CHECK(clSetKernelArg(config->kernel[kernel_idx], 1, sizeof(cl_mem), ACC_OPENCL_MEM(dev_a_data)),
          "set A-matrix argument of SMM-kernel", result);
        ACC_OPENCL_CHECK(clSetKernelArg(config->kernel[kernel_idx], 2, sizeof(cl_mem), ACC_OPENCL_MEM(dev_b_data)),
          "set B-matrix argument of SMM-kernel", result);
        ACC_OPENCL_CHECK(clSetKernelArg(config->kernel[kernel_idx], 3, sizeof(cl_mem), ACC_OPENCL_MEM(dev_param_stack)),
          "set batch-list argument of SMM-kernel", result);
        if (0 == kernel_idx) {
          assert(bs <= config->bs);
          ACC_OPENCL_CHECK(clSetKernelArg(config->kernel[kernel_idx], 4, sizeof(int), &stack_size),
            "set stacksize argument of SMM-kernel", result);
          ACC_OPENCL_CHECK(
            clSetKernelArg(config->kernel[kernel_idx], 5, sizeof(int), &bs), "set minibatch argument of SMM-kernel", result);
        }
        ACC_OPENCL_CHECK(clEnqueueNDRangeKernel(queue, config->kernel[kernel_idx], 1 /*work_dim*/, NULL /*offset*/, &work_size,
                           config->wgsize + kernel_idx, 0, NULL, perf_event),
          "launch SMM-kernel", result);
        /* eventually update performance counters inside of locked region */
#    if !defined(OPENCL_LIBSMM_VALIDATE_SMM)
        if (3 <= c_dbcsr_acc_opencl_config.verbosity || 0 > c_dbcsr_acc_opencl_config.verbosity) {
          if (NULL != perf_event) {
            cl_ulong begin = 0, end = 0;
            clWaitForEvents(1, perf_event);
            ACC_OPENCL_CHECK(clGetEventProfilingInfo(*perf_event, CL_PROFILING_COMMAND_START, sizeof(cl_ulong), &begin, NULL),
              "query kernel start time", result);
            ACC_OPENCL_CHECK(clGetEventProfilingInfo(*perf_event, CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &end, NULL),
              "query kernel end time", result);
            duration = 1E-9 * LIBXSMM_DELTA(begin, end); /* Nanoseconds->seconds */
          }
          else {
            clFinish(queue);
            duration = libxsmm_timer_duration(start, libxsmm_timer_tick()); /* seconds */
          }
          if (EXIT_SUCCESS == result) {
            const double gflops = 1E-9 * (2ULL * m_max * n_max * k_max * stack_size) / duration;
            const double est = (dbcsr_type_real_8 == datatype
                                  ? (OPENCL_LIBSMM_AI(m_max, n_max, k_max, sizeof(double)) * opencl_libsmm_dacc)
                                  : (OPENCL_LIBSMM_AI(m_max, n_max, k_max, sizeof(float)) * opencl_libsmm_sacc));
            const int* const priority = c_dbcsr_acc_opencl_stream_priority(stream);
            LIBXSMM_STDIO_ACQUIRE();
            fprintf(stderr, "INFO ACC/OpenCL: SMM-kernel ");
            opencl_libsmm_write_smm_params(
              stderr, 1 /*only_key*/, &key, NULL /*config*/, NULL /*delim*/, NULL /*begin*/, NULL /*close*/);
            fprintf(stderr, "=");
            opencl_libsmm_write_smm_params(stderr, 1 /*only_key*/, &key, config, NULL /*delim*/, NULL /*begin*/, NULL /*close*/);
            fprintf(stderr, " prio=%i ss=%i cur=%.1f", NULL != priority ? *priority : -1, stack_size, gflops);
            if (0 < est) fprintf(stderr, " est=%.1f", est);
            fprintf(stderr, " GFLOPS/s dur=%.2g ms\n", 1E3 * duration);
            LIBXSMM_STDIO_RELEASE();
          }
        }
#    endif
#    if defined(OPENCL_LIBSMM_VALIDATE_SMM)
        ACC_OPENCL_CHECK(c_dbcsr_acc_memcpy_d2h(dev_c_data, test, csize, stream), "transfer debug test", result);
#    endif
#    if defined(OPENCL_LIBSMM_VALIDATE_SMM)
        ACC_OPENCL_CHECK(c_dbcsr_acc_stream_sync(stream), "sync stream", result);
#    endif
#    if defined(OPENCL_LIBSMM_VALIDATE_SMM)
        if (EXIT_SUCCESS == result) {
          const char* const env_tol = getenv("OPENCL_LIBSMM_SMM_TOLERANCE");
          const double tolerance = ((NULL == env_tol || '\0' == *env_tol) ? 1E-3 : atof(env_tol));
          const int* const params = host_param_stack + (4 <= nparams ? (nparams - 4) : 0);
          size_t i;
          LIBXSMM_STDIO_ACQUIRE();
          if (0 != c_dbcsr_acc_opencl_config.verbosity) {
            fprintf(stderr, "libsmm_acc_process(size=%i, type=%s, m=%i, n=%i, k=%i, max=%i, stream=%p)", stack_size,
              dbcsr_type_real_8 == datatype ? "f64" : (dbcsr_type_real_4 == datatype ? "f32" : "unknown"), m_max, n_max, k_max,
              max_kernel_dim, stream);
          }
          for (i = 0; i < ((size_t)stack_size * nparams); i += nparams) {
            const size_t ia = (size_t)(params[i + 0] - 1) * typesize;
            const size_t ib = (size_t)(params[i + 1] - 1) * typesize;
            const size_t ic = (size_t)(params[i + 2] - 1) * typesize;
            assert(ia < asize && ib < bsize && ic < csize);
            libxsmm_otrans(btrn, binp + ib, typesize, n_max, k_max, n_max, k_max);
            kernel_cpu.xmm(ainp + ia, btrn, gold + ic);
          }
          /* some result may be validated multiple times in case of duplicated c-indexes */
          for (i = 0; i < ((size_t)stack_size * nparams); i += nparams) {
            const size_t ic = (size_t)(params[i + 2] - 1) * typesize;
            libxsmm_matdiff_info diff;
            libxsmm_matdiff(
              &diff, (libxsmm_datatype)precision, m_max, n_max, gold + ic, test + ic, &m_max /*ldref*/, &m_max /*ldtst*/);
            if (tolerance < diff.normf_rel) {
              if (0 == c_dbcsr_acc_opencl_config.verbosity) {
                fprintf(stderr, "libsmm_acc_process(size=%i, type=%s, m=%i, n=%i, k=%i, max=%i, stream=%p)", stack_size,
                  dbcsr_type_real_8 == datatype ? "f64" : (dbcsr_type_real_4 == datatype ? "f32" : "unknown"), m_max, n_max, k_max,
                  max_kernel_dim, stream);
              }
#      if LIBXSMM_VERSION4(1, 17, 0, 0) < LIBXSMM_VERSION_NUMBER
              fprintf(stderr, " => ERROR diff=%g (%g != %g)\n", diff.linf_abs, diff.v_ref, diff.v_tst);
#      else
              fprintf(stderr, " => ERROR diff=%g\n", diff.linf_abs);
#      endif
#      if defined(_DEBUG)
              opencl_libsmm_print_matrix(stderr, "gold = ", datatype, gold + ic, m_max, n_max);
              opencl_libsmm_print_matrix(stderr, "test = ", datatype, test + ic, m_max, n_max);
              fprintf(stderr, "\n");
#      endif
#      if defined(OPENCL_LIBSMM_VALIDATE_EXIT)
              exit(EXIT_FAILURE);
#      else
              result = EXIT_FAILURE;
              break;
#      endif
            }
          }
          if (0 != c_dbcsr_acc_opencl_config.verbosity && EXIT_SUCCESS == result) {
            fprintf(stderr, " => OK\n");
          }
          LIBXSMM_STDIO_RELEASE();
        }
        libxsmm_free(scratch);
#    elif defined(NDEBUG)
        LIBXSMM_UNUSED(host_param_stack);
        LIBXSMM_UNUSED(nparams);
#    endif
      }
      LIBXSMM_ATOMIC_RELEASE(lock, LIBXSMM_ATOMIC_RELAXED);
    }
  }
  else if (0 < stack_size) { /* inhomogeneous, large kernel, or unsupported datatype */
    return -1; /* TODO: document result code to trigger host-fallback */
  }
#  endif
  ACC_OPENCL_RETURN(result);
}


int c_calculate_norms(double* mat, int nblks, int* offsets, int* nelems, float* norms, void* stream_ptr) {
  LIBXSMM_UNUSED(mat);
  LIBXSMM_UNUSED(nblks);
  LIBXSMM_UNUSED(offsets);
  LIBXSMM_UNUSED(nelems);
  LIBXSMM_UNUSED(norms);
  LIBXSMM_UNUSED(stream_ptr);
  return -1;
}

#  if defined(__cplusplus)
}
#  endif

#endif /*__OPENCL*/
