/*
     This file is part of GNUnet
     Copyright (C) 2018 GNUnet e.V.

     GNUnet is free software: you can redistribute it and/or modify it
     under the terms of the GNU Affero General Public License as published
     by the Free Software Foundation, either version 3 of the License,
     or (at your option) any later version.

     GNUnet is distributed in the hope that it will be useful, but
     WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     Affero General Public License for more details.

     You should have received a copy of the GNU Affero General Public License
     along with this program.  If not, see <http://www.gnu.org/licenses/>.

     SPDX-License-Identifier: AGPL3.0-or-later
 */
/**
 * @file src/namestore/gnunet-zoneimport.c
 * @brief import a DNS zone for publication in GNS, incremental
 * @author Christian Grothoff
 */
#include "platform.h"
#if HAVE_LIBIDN2
#if HAVE_IDN2_H
#include <idn2.h>
#elif HAVE_IDN2_IDN2_H
#include <idn2/idn2.h>
#endif
#elif HAVE_LIBIDN
#if HAVE_IDNA_H
#include <idna.h>
#elif HAVE_IDN_IDNA_H
#include <idn/idna.h>
#endif
#endif
#include <gnunet_util_lib.h>
#include <gnunet_gnsrecord_lib.h>
#include <gnunet_namestore_service.h>
#include <gnunet_statistics_service.h>
#include <gnunet_identity_service.h>


/**
 * Maximum number of queries pending at the same time.
 */
#define THRESH 100

/**
 * TIME_THRESH is in usecs.  How quickly do we submit fresh queries.
 * Used as an additional throttle.
 */
#define TIME_THRESH 10

/**
 * How often do we retry a query before giving up for good?
 */
#define MAX_RETRIES 5

/**
 * How many DNS requests do we at most issue in rapid series?
 */
#define MAX_SERIES 10

/**
 * How long do we wait at least between series of requests?
 */
#define SERIES_DELAY \
        GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10)

/**
 * How long do DNS records have to last at least after being imported?
 */
static struct GNUNET_TIME_Relative minimum_expiration_time;

/**
 * How many requests do we request from NAMESTORE in one batch
 * during our initial iteration?
 */
#define NS_BATCH_SIZE 1024

/**
 * Some zones may include authoritative records for other
 * zones, such as foo.com.uk or bar.com.fr.  As for GNS
 * each dot represents a zone cut, we then need to create a
 * zone on-the-fly to capture those records properly.
 */
struct Zone
{
  /**
   * Kept in a DLL.
   */
  struct Zone *next;

  /**
   * Kept in a DLL.
   */
  struct Zone *prev;

  /**
   * Domain of the zone (i.e. "fr" or "com.fr")
   */
  char *domain;

  /**
   * Private key of the zone.
   */
  struct GNUNET_CRYPTO_BlindablePrivateKey key;
};


/**
 * Record for the request to be stored by GNS.
 */
struct Record
{
  /**
   * Kept in a DLL.
   */
  struct Record *next;

  /**
   * Kept in a DLL.
   */
  struct Record *prev;

  /**
   * GNS record.
   */
  struct GNUNET_GNSRECORD_Data grd;
};


/**
 * Request we should make.  We keep this struct in memory per request,
 * thus optimizing it is crucial for the overall memory consumption of
 * the zone importer.
 */
struct Request
{
  /**
   * Requests are kept in a heap while waiting to be resolved.
   */
  struct GNUNET_CONTAINER_HeapNode *hn;

  /**
   * Active requests are kept in a DLL.
   */
  struct Request *next;

  /**
   * Active requests are kept in a DLL.
   */
  struct Request *prev;

  /**
   * Head of records that should be published in GNS for
   * this hostname.
   */
  struct Record *rec_head;

  /**
   * Tail of records that should be published in GNS for
   * this hostname.
   */
  struct Record *rec_tail;

  /**
   * Socket used to make the request, NULL if not active.
   */
  struct GNUNET_DNSSTUB_RequestSocket *rs;

  /**
   * Hostname we are resolving, allocated at the end of
   * this struct (optimizing memory consumption by reducing
   * total number of allocations).
   */
  char *hostname;

  /**
   * Namestore operation pending for this record.
   */
  struct GNUNET_NAMESTORE_QueueEntry *qe;

  /**
   * Zone responsible for this request.
   */
  const struct Zone *zone;

  /**
   * At what time does the (earliest) of the returned records
   * for this name expire? At this point, we need to re-fetch
   * the record.
   */
  struct GNUNET_TIME_Absolute expires;

  /**
   * While we are fetching the record, the value is set to the
   * starting time of the DNS operation.  While doing a
   * NAMESTORE store, again set to the start time of the
   * NAMESTORE operation.
   */
  struct GNUNET_TIME_Absolute op_start_time;

  /**
   * How often did we issue this query? (And failed, reset
   * to zero once we were successful.)
   */
  unsigned int issue_num;

  /**
   * random 16-bit DNS query identifier.
   */
  uint16_t id;
};

/**
 * Missing identity creation context
 */
struct MissingZoneCreationCtx
{
  // DLL
  struct MissingZoneCreationCtx *next;

  // DLL
  struct MissingZoneCreationCtx *prev;

  // Operation
  struct GNUNET_IDENTITY_Operation *id_op;

  // NS Operation
  struct GNUNET_NAMESTORE_QueueEntry *ns_qe;

  // Request
  struct Request *req;
};

// Missing zones list
static struct MissingZoneCreationCtx *missing_zones_head;

// Missing zones list
static struct MissingZoneCreationCtx *missing_zones_tail;

// Lines provided on stdin
static uint64_t stdin_lines_read;

/**
 * Command-line argument specifying desired size of the hash map with
 * all of our pending names.  Usually, we use an automatically growing
 * map, but this is only OK up to about a million entries.  Above that
 * number, the user must explicitly specify the size at startup.
 */
static unsigned int map_size = 1024;

/**
 * Handle to the identity service.
 */
static struct GNUNET_IDENTITY_Handle *id;

/**
 * Namestore handle.
 */
static struct GNUNET_NAMESTORE_Handle *ns;

/**
 * Handle to the statistics service.
 */
static struct GNUNET_STATISTICS_Handle *stats;

/**
 * Context for DNS resolution.
 */
static struct GNUNET_DNSSTUB_Context *dns_ctx;

/**
 * The number of DNS queries that are outstanding
 */
static unsigned int pending;

/**
 * The number of NAMESTORE record store operations that are outstanding
 */
static unsigned int pending_rs;

/**
 * Number of lookups we performed overall.
 */
static unsigned int lookups;

/**
 * Number of records we had cached.
 */
static unsigned int cached;

/**
 * How many hostnames did we reject (malformed).
 */
static unsigned int rejects;

/**
 * Number of lookups that failed.
 */
static unsigned int failures;

/**
 * Number of records we found.
 */
static unsigned int records;

/**
 * Number of record sets given to namestore.
 */
static unsigned int record_sets;

/**
 * Heap of all requests to perform, sorted by
 * the time we should next do the request (i.e. by expires).
 */
static struct GNUNET_CONTAINER_Heap *req_heap;

/**
 * Active requests are kept in a DLL.
 */
static struct Request *req_head;

/**
 * Active requests are kept in a DLL.
 */
static struct Request *req_tail;

/**
 * Main task.
 */
static struct GNUNET_SCHEDULER_Task *t;

/**
 * Hash map of requests for which we may still get a response from
 * the namestore.  Set to NULL once the initial namestore iteration
 * is done.
 */
static struct GNUNET_CONTAINER_MultiHashMap *ns_pending;

/**
 * Current zone iteration handle.
 */
static struct GNUNET_NAMESTORE_ZoneIterator *zone_it;

/**
 * Head of list of zones we are managing.
 */
static struct Zone *zone_head;

/**
 * Tail of list of zones we are managing.
 */
static struct Zone *zone_tail;

/**
 * After how many more results must #ns_lookup_result_cb() ask
 * the namestore for more?
 */
static uint64_t ns_iterator_trigger_next;

/**
 * Number of DNS requests counted in latency total.
 */
static uint64_t total_dns_latency_cnt;

/**
 * Sum of DNS latencies observed.
 */
static struct GNUNET_TIME_Relative total_dns_latency;

/**
 * Number of records processed (DNS lookup, no NAMESTORE) in total.
 */
static uint64_t total_reg_proc_dns;

/**
 * Number of records processed (DNS lookup, with NAMESTORE) in total.
 */
static uint64_t total_reg_proc_dns_ns;

/**
 * Start time of the regular processing.
 */
static struct GNUNET_TIME_Absolute start_time_reg_proc;

/**
 * Last time we worked before going idle.
 */
static struct GNUNET_TIME_Absolute sleep_time_reg_proc;

/**
 * Time we slept just waiting for work.
 */
static struct GNUNET_TIME_Relative idle_time;


/**
 * Callback for #for_all_records
 *
 * @param cls closure
 * @param rec a DNS record
 */
typedef void (*RecordProcessor) (void *cls,
                                 const struct GNUNET_DNSPARSER_Record *rec);


/**
 * Call @a rp for each record in @a p, regardless of
 * what response section it is in.
 *
 * @param p packet from DNS
 * @param rp function to call
 * @param rp_cls closure for @a rp
 */
static void
for_all_records (const struct GNUNET_DNSPARSER_Packet *p,
                 RecordProcessor rp,
                 void *rp_cls)
{
  for (unsigned int i = 0; i < p->num_answers; i++)
  {
    struct GNUNET_DNSPARSER_Record *rs = &p->answers[i];

    rp (rp_cls, rs);
  }
  for (unsigned int i = 0; i < p->num_authority_records; i++)
  {
    struct GNUNET_DNSPARSER_Record *rs = &p->authority_records[i];

    rp (rp_cls, rs);
  }
  for (unsigned int i = 0; i < p->num_additional_records; i++)
  {
    struct GNUNET_DNSPARSER_Record *rs = &p->additional_records[i];

    rp (rp_cls, rs);
  }
}


/**
 * Return just the label of the hostname in @a req.
 *
 * @param req request to process hostname of
 * @return statically allocated pointer to the label,
 *         overwritten upon the next request!
 */
static const char *
get_label (struct Request *req)
{
  static char label[64];
  const char *dot;

  dot = strchr (req->hostname, (unsigned char) '.');
  if (NULL == dot)
  {
    GNUNET_break (0);
    return NULL;
  }
  if (((size_t) (dot - req->hostname)) >= sizeof(label))
  {
    GNUNET_break (0);
    return NULL;
  }
  GNUNET_memcpy (label, req->hostname, dot - req->hostname);
  label[dot - req->hostname] = '\0';

  return label;
}


static char*
get_label_utf8 (struct Request *req)
{
  char *label_utf8;
  const char *label_ace;

  label_ace = get_label (req);
  idna_to_unicode_8z8z (label_ace,
                        &label_utf8,
                        IDNA_ALLOW_UNASSIGNED);
  return label_utf8;
}


/**
 * Build DNS query for @a hostname.
 *
 * @param hostname host to build query for
 * @param[out] raw_size number of bytes in the query
 * @return NULL on error, otherwise pointer to statically (!)
 *         allocated query buffer
 */
static void *
build_dns_query (struct Request *req, size_t *raw_size)
{
  static char raw[512];
  char *name_ace;
  char *rawp;
  struct GNUNET_DNSPARSER_Packet p;
  struct GNUNET_DNSPARSER_Query q;
  int ret;

  if (IDNA_SUCCESS != idna_to_ascii_8z (req->hostname,
                                        &name_ace,
                                        IDNA_ALLOW_UNASSIGNED))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _ ("Name `%s' cannot be converted to IDNA."),
                req->hostname);
    return NULL;
  }
  q.name = (char *) name_ace;
  q.type = GNUNET_DNSPARSER_TYPE_NS;
  q.dns_traffic_class = GNUNET_TUN_DNS_CLASS_INTERNET;

  memset (&p, 0, sizeof(p));
  p.num_queries = 1;
  p.queries = &q;
  p.id = req->id;
  ret = GNUNET_DNSPARSER_pack (&p, UINT16_MAX, &rawp, raw_size);
  if (GNUNET_OK != ret)
  {
    if (GNUNET_NO == ret)
      GNUNET_free (rawp);
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to pack query for hostname `%s'\n",
                name_ace);
    rejects++;
    GNUNET_free (name_ace);
    return NULL;
  }
  if (*raw_size > sizeof(raw))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to pack query for hostname `%s'\n",
                name_ace);
    rejects++;
    GNUNET_break (0);
    GNUNET_free (name_ace);
    GNUNET_free (rawp);
    return NULL;
  }
  GNUNET_memcpy (raw, rawp, *raw_size);
  GNUNET_free (rawp);
  return raw;
}


/**
 * Free records associated with @a req.
 *
 * @param req request to free records of
 */
static void
free_records (struct Request *req)
{
  struct Record *rec;

  /* Free records */
  while (NULL != (rec = req->rec_head))
  {
    GNUNET_CONTAINER_DLL_remove (req->rec_head, req->rec_tail, rec);
    GNUNET_free (rec);
  }
}


/**
 * Free @a req and data structures reachable from it.
 *
 * @param req request to free
 */
static void
free_request (struct Request *req)
{
  free_records (req);
  GNUNET_free (req);
}


/**
 * Process as many requests as possible from the queue.
 *
 * @param cls NULL
 */
static void
process_queue (void *cls);


/**
 * Insert @a req into DLL sorted by next fetch time.
 *
 * @param req request to insert into #req_heap
 */
static void
insert_sorted (struct Request *req)
{
  req->hn =
    GNUNET_CONTAINER_heap_insert (req_heap, req, req->expires.abs_value_us);
  if (req == GNUNET_CONTAINER_heap_peek (req_heap))
  {
    if (NULL != t)
      GNUNET_SCHEDULER_cancel (t);
    sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
    t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL);
  }
}


/**
 * Add record to the GNS record set for @a req.
 *
 * @param req the request to expand GNS record set for
 * @param type type to use
 * @param expiration_time when should @a rec expire
 * @param data raw data to store
 * @param data_len number of bytes in @a data
 */
static void
add_record (struct Request *req,
            uint32_t type,
            struct GNUNET_TIME_Absolute expiration_time,
            const void *data,
            size_t data_len)
{
  struct Record *rec;

  rec = GNUNET_malloc (sizeof(struct Record) + data_len);
  rec->grd.data = &rec[1];
  rec->grd.expiration_time = expiration_time.abs_value_us;
  rec->grd.data_size = data_len;
  rec->grd.record_type = type;
  rec->grd.flags = GNUNET_GNSRECORD_RF_NONE;
  GNUNET_memcpy (&rec[1], data, data_len);
  GNUNET_CONTAINER_DLL_insert (req->rec_head, req->rec_tail, rec);
}


/**
 * Closure for #check_for_glue.
 */
struct GlueClosure
{
  /**
   * Overall request we are processing.
   */
  struct Request *req;

  /**
   * NS name we are looking for glue for.
   */
  const char *ns;

  /**
   * Set to #GNUNET_YES if glue was found.
   */
  int found;
};


/**
 * Try to find glue records for a given NS record.
 *
 * @param cls a `struct GlueClosure *`
 * @param rec record that may contain glue information
 */
static void
check_for_glue (void *cls, const struct GNUNET_DNSPARSER_Record *rec)
{
  struct GlueClosure *gc = cls;
  char dst[65536];
  size_t rd_len;
  char ip[INET6_ADDRSTRLEN + 1];
  socklen_t ip_size = (socklen_t) sizeof(ip);
  struct GNUNET_TIME_Absolute expiration_time;
  struct GNUNET_TIME_Relative left;

  if (0 != strcasecmp (rec->name, gc->ns))
    return;
  expiration_time = rec->expiration_time;
  left = GNUNET_TIME_absolute_get_remaining (expiration_time);
  if (0 == left.rel_value_us)
    return; /* ignore expired glue records */
  /* if expiration window is too short, bump it to configured minimum */
  if (left.rel_value_us < minimum_expiration_time.rel_value_us)
    expiration_time =
      GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
  rd_len = 0;
  switch (rec->type)
  {
  case GNUNET_DNSPARSER_TYPE_A:
    if (sizeof(struct in_addr) != rec->data.raw.data_len)
    {
      GNUNET_break (0);
      return;
    }
    if (NULL == inet_ntop (AF_INET, rec->data.raw.data, ip, ip_size))
    {
      GNUNET_break (0);
      return;
    }
    // NAME
    GNUNET_memcpy (dst,
                   gc->req->hostname,
                   strlen (gc->req->hostname) + 1);
    rd_len += strlen (gc->req->hostname) + 1;
    // DNS SERVER NAME
    GNUNET_memcpy (dst + strlen (gc->req->hostname) + 1,
                   ip,
                   strlen (ip) + 1);
    rd_len += strlen (ip) + 1;
    add_record (gc->req,
                GNUNET_GNSRECORD_TYPE_GNS2DNS,
                expiration_time,
                dst,
                rd_len);
    gc->found = GNUNET_YES;
    break;

  case GNUNET_DNSPARSER_TYPE_AAAA:
    if (sizeof(struct in6_addr) != rec->data.raw.data_len)
    {
      GNUNET_break (0);
      return;
    }
    if (NULL == inet_ntop (AF_INET6, rec->data.raw.data, ip, ip_size))
    {
      GNUNET_break (0);
      return;
    }
    // NAME
    GNUNET_memcpy (dst,
                   gc->req->hostname,
                   strlen (gc->req->hostname) + 1);
    rd_len += strlen (gc->req->hostname) + 1;
    // DNS SERVER NAME
    GNUNET_memcpy (dst + strlen (gc->req->hostname) + 1,
                   ip,
                   strlen (ip) + 1);
    rd_len += strlen (ip) + 1;
    add_record (gc->req,
                GNUNET_GNSRECORD_TYPE_GNS2DNS,
                expiration_time,
                dst,
                rd_len);
    gc->found = GNUNET_YES;
    break;

  case GNUNET_DNSPARSER_TYPE_CNAME:
    // NAME
    GNUNET_memcpy (dst,
                   gc->req->hostname,
                   strlen (gc->req->hostname) + 1);
    rd_len += strlen (gc->req->hostname) + 1;
    // DNS SERVER NAME
    GNUNET_memcpy (dst + strlen (gc->req->hostname) + 1,
                   rec->data.hostname,
                   strlen (rec->data.hostname) + 1);
    rd_len += strlen (rec->data.hostname) + 1;
    add_record (gc->req,
                GNUNET_GNSRECORD_TYPE_GNS2DNS,
                expiration_time,
                dst,
                rd_len);
    gc->found = GNUNET_YES;
    break;

  default:
    /* useless, do nothing */
    break;
  }
}


/**
 * Closure for #process_record().
 */
struct ProcessRecordContext
{
  /**
   * Answer we got back and are currently parsing, or NULL
   * if not active.
   */
  struct GNUNET_DNSPARSER_Packet *p;

  /**
   * Request we are processing.
   */
  struct Request *req;
};


/**
 * We received @a rec for @a req. Remember the answer.
 *
 * @param cls a `struct ProcessRecordContext`
 * @param rec response
 */
static void
process_record (void *cls, const struct GNUNET_DNSPARSER_Record *rec)
{
  struct ProcessRecordContext *prc = cls;
  struct Request *req = prc->req;
  char dst[65536];
  size_t dst_len;
  size_t off;
  struct GNUNET_TIME_Absolute expiration_time;
  struct GNUNET_TIME_Relative left;

  dst_len = sizeof(dst);
  off = 0;
  records++;
  if (0 != strcasecmp (rec->name, req->hostname))
  {
    GNUNET_log (
      GNUNET_ERROR_TYPE_DEBUG,
      "DNS returned record from zone `%s' of type %u while resolving `%s'\n",
      rec->name,
      (unsigned int) rec->type,
      req->hostname);
    return;   /* does not match hostname, might be glue, but
                 not useful for this pass! */
  }
  expiration_time = rec->expiration_time;
  left = GNUNET_TIME_absolute_get_remaining (expiration_time);
  if (0 == left.rel_value_us)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "DNS returned expired record for `%s'\n",
                req->hostname);
    GNUNET_STATISTICS_update (stats,
                              "# expired records obtained from DNS",
                              1,
                              GNUNET_NO);
    return;   /* record expired */
  }

  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "DNS returned record that expires at %s for `%s'\n",
              GNUNET_STRINGS_absolute_time_to_string (expiration_time),
              req->hostname);
  /* if expiration window is too short, bump it to configured minimum */
  if (left.rel_value_us < minimum_expiration_time.rel_value_us)
    expiration_time =
      GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
  switch (rec->type)
  {
  case GNUNET_DNSPARSER_TYPE_NS: {
      struct GlueClosure gc;

      /* check for glue */
      gc.req = req;
      gc.ns = rec->data.hostname;
      gc.found = GNUNET_NO;
      for_all_records (prc->p, &check_for_glue, &gc);
      if (GNUNET_NO == gc.found)
      {
        // NAME
        GNUNET_memcpy (dst,
                       req->hostname,
                       strlen (req->hostname) + 1);
        off += strlen (req->hostname) + 1;
        // DNS SERVER NAME
        GNUNET_memcpy (dst + strlen (req->hostname) + 1,
                       rec->data.hostname,
                       strlen (rec->data.hostname) + 1);
        off += strlen (rec->data.hostname) + 1;
        /* FIXME: actually check if this is out-of-bailiwick,
             and if not request explicit resolution... */
        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                    "Converted OOB (`%s') NS record for `%s'\n",
                    rec->data.hostname,
                    rec->name);
        add_record (req,
                    GNUNET_GNSRECORD_TYPE_GNS2DNS,
                    expiration_time,
                    dst,
                    off);
      }
      else
      {
        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                    "Converted NS record for `%s' using glue\n",
                    rec->name);
      }
      break;
    }

  case GNUNET_DNSPARSER_TYPE_CNAME:
    if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
                                                        dst_len,
                                                        &off,
                                                        rec->data.hostname))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting CNAME (`%s') record for `%s'\n",
                  rec->data.hostname,
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;

  case GNUNET_DNSPARSER_TYPE_DNAME:
    /* No support for DNAME in GNS yet! FIXME: support later! */
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                "FIXME: not supported: %s DNAME %s\n",
                rec->name,
                rec->data.hostname);
    break;

  case GNUNET_DNSPARSER_TYPE_MX:
    if (GNUNET_OK ==
        GNUNET_DNSPARSER_builder_add_mx (dst, dst_len, &off, rec->data.mx))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting MX (`%s') record for `%s'\n",
                  rec->data.mx->mxhost,
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;

  case GNUNET_DNSPARSER_TYPE_SOA:
    if (GNUNET_OK ==
        GNUNET_DNSPARSER_builder_add_soa (dst, dst_len, &off, rec->data.soa))
    {
      /* NOTE: GNS does not really use SOAs */
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting SOA record for `%s'\n",
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;

  case GNUNET_DNSPARSER_TYPE_SRV:
    if (GNUNET_OK ==
        GNUNET_DNSPARSER_builder_add_srv (dst, dst_len, &off, rec->data.srv))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting SRV record for `%s'\n",
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;

  case GNUNET_DNSPARSER_TYPE_URI:
    if (GNUNET_OK ==
        GNUNET_DNSPARSER_builder_add_uri (dst, dst_len, &off, rec->data.uri))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting URI record for `%s'\n",
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;

  case GNUNET_DNSPARSER_TYPE_PTR:
    if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
                                                        dst_len,
                                                        &off,
                                                        rec->data.hostname))
    {
      /* !?: what does a PTR record do in a regular TLD??? */
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting PTR record for `%s' (weird)\n",
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;

  case GNUNET_DNSPARSER_TYPE_CERT:
    if (GNUNET_OK ==
        GNUNET_DNSPARSER_builder_add_cert (dst, dst_len, &off, rec->data.cert))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting CERT record for `%s'\n",
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;

  /* Rest is 'raw' encoded and just needs to be copied IF
     the hostname matches the requested name; otherwise we
     simply cannot use it. */
  case GNUNET_DNSPARSER_TYPE_A:
  case GNUNET_DNSPARSER_TYPE_AAAA:
  case GNUNET_DNSPARSER_TYPE_TXT:
  default:
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Converting record of type %u for `%s'\n",
                (unsigned int) rec->type,
                rec->name);
    add_record (req,
                rec->type,
                expiration_time,
                rec->data.raw.data,
                rec->data.raw.data_len);
    break;
  }
}


static void
store_completed_cb (void *cls, enum GNUNET_ErrorCode ec)
{
  static struct GNUNET_TIME_Absolute last;
  struct Request *req = cls;

  req->qe = NULL;
  if (GNUNET_EC_NONE != ec)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to store zone data for `%s': %s\n",
                req->hostname,
                GNUNET_ErrorCode_get_hint (ec));
  }
  else
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Stored records under `%s' (%d)\n",
                req->hostname,
                ec);
  }
  total_reg_proc_dns_ns++; /* finished regular processing */
  pending_rs--;
  free_records (req);
  /* compute NAMESTORE statistics */
  {
    static uint64_t total_ns_latency_cnt;
    static struct GNUNET_TIME_Relative total_ns_latency;
    struct GNUNET_TIME_Relative ns_latency;

    ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
    total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency, ns_latency);
    if (0 == total_ns_latency_cnt)
      last = GNUNET_TIME_absolute_get ();
    total_ns_latency_cnt++;
    if (0 == (total_ns_latency_cnt % 1000))
    {
      struct GNUNET_TIME_Relative delta;
      struct GNUNET_TIME_Relative est_remaining;

      delta = GNUNET_TIME_absolute_get_duration (last);
      last = GNUNET_TIME_absolute_get ();
      est_remaining = GNUNET_TIME_relative_multiply (
        delta,
        (stdin_lines_read - total_ns_latency_cnt) / 1000);
      fprintf (stderr,
               "Processed 1000 records in %s. ",
               GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES));
      fprintf (stderr,
               "Estimated remaining time: %s. ",
               GNUNET_STRINGS_relative_time_to_string (
                 est_remaining,
                 GNUNET_YES));
      fprintf (stderr,
               "Last domain processed: %s\n",
               req->hostname);
      GNUNET_STATISTICS_set (stats,
                             "# average NAMESTORE PUT latency (μs)",
                             total_ns_latency.rel_value_us
                             / total_ns_latency_cnt,
                             GNUNET_NO);
    }
  }
  /* compute and publish overall velocity */
  if (0 == (total_reg_proc_dns_ns % 100))
  {
    struct GNUNET_TIME_Relative runtime;

    runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc);
    runtime = GNUNET_TIME_relative_subtract (runtime, idle_time);
    runtime =
      GNUNET_TIME_relative_divide (runtime,
                                   total_reg_proc_dns + total_reg_proc_dns_ns);
    GNUNET_STATISTICS_set (stats,
                           "# Regular processing completed without NAMESTORE",
                           total_reg_proc_dns,
                           GNUNET_NO);
    GNUNET_STATISTICS_set (stats,
                           "# Regular processing completed with NAMESTORE PUT",
                           total_reg_proc_dns_ns,
                           GNUNET_NO);
    GNUNET_STATISTICS_set (stats,
                           "# average request processing latency (μs)",
                           runtime.rel_value_us,
                           GNUNET_NO);
    GNUNET_STATISTICS_set (stats,
                           "# total time spent idle (μs)",
                           idle_time.rel_value_us,
                           GNUNET_NO);
  }

  if (NULL == t)
  {
    sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
    t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
  }
}


/**
 * Function called with the result of a DNS resolution.
 *
 * @param cls closure with the `struct Request`
 * @param dns dns response, never NULL
 * @param dns_len number of bytes in @a dns
 */
static void
process_result (void *cls,
                const struct GNUNET_TUN_DnsHeader *dns,
                size_t dns_len)
{
  struct Request *req = cls;
  struct Record *rec;
  struct GNUNET_DNSPARSER_Packet *p;
  unsigned int rd_count;

  GNUNET_assert (NULL == req->hn);
  if ((NULL == dns) ||
      (dns->flags.return_code != GNUNET_TUN_DNS_RETURN_CODE_NO_ERROR))
  {
    /* stub gave up */
    GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
    if (NULL != dns)
    {
      GNUNET_DNSSTUB_resolve_cancel (req->rs);
      req->rs = NULL;
    }
    pending--;
    if (NULL == t)
    {
      sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
      t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
    }
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Stub gave up on DNS reply for `%s'\n",
                req->hostname);
    GNUNET_STATISTICS_update (stats, "# Unsuccessful DNS lookups", 1, GNUNET_NO)
    ;
    if (req->issue_num > MAX_RETRIES)
    {
      failures++;
      free_request (req);
      GNUNET_STATISTICS_update (stats, "# DNS requests given up on", 1,
                                GNUNET_NO);
      return;
    }
    total_reg_proc_dns++;
    req->rs = NULL;
    insert_sorted (req);
    return;
  }
  if (req->id != dns->id)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "DNS ID did not match request, ignoring reply\n");
    GNUNET_STATISTICS_update (stats, "# DNS ID mismatches", 1, GNUNET_NO);
    return;
  }
  GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
  GNUNET_DNSSTUB_resolve_cancel (req->rs);
  req->rs = NULL;
  pending--;
  p = GNUNET_DNSPARSER_parse ((const char *) dns, dns_len);
  if (NULL == p)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to parse DNS reply for `%s'\n",
                req->hostname);
    GNUNET_STATISTICS_update (stats, "# DNS parser errors", 1, GNUNET_NO);
    if (NULL == t)
    {
      sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
      t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
    }
    if (req->issue_num > MAX_RETRIES)
    {
      failures++;
      free_request (req);
      GNUNET_STATISTICS_update (stats, "# requests given up on", 1, GNUNET_NO);
      return;
    }
    insert_sorted (req);
    return;
  }
  /* import new records */
  req->issue_num = 0; /* success, reset counter! */
  {
    struct ProcessRecordContext prc = { .req = req, .p = p };

    for_all_records (p, &process_record, &prc);
  }
  GNUNET_DNSPARSER_free_packet (p);
  /* count records found, determine minimum expiration time */
  req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
  {
    struct GNUNET_TIME_Relative dns_latency;

    dns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
    total_dns_latency =
      GNUNET_TIME_relative_add (total_dns_latency, dns_latency);
    total_dns_latency_cnt++;
    if (0 == (total_dns_latency_cnt % 1000))
    {
      GNUNET_STATISTICS_set (stats,
                             "# average DNS lookup latency (μs)",
                             total_dns_latency.rel_value_us
                             / total_dns_latency_cnt,
                             GNUNET_NO);
    }
  }
  rd_count = 0;
  for (rec = req->rec_head; NULL != rec; rec = rec->next)
  {
    struct GNUNET_TIME_Absolute at;

    at.abs_value_us = rec->grd.expiration_time;
    req->expires = GNUNET_TIME_absolute_min (req->expires, at);
    rd_count++;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Obtained %u records for `%s'\n",
              rd_count,
              req->hostname);
  /* Instead of going for SOA, simplified for now to look each
     day in case we got an empty response */
  if (0 == rd_count)
  {
    req->expires = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS);
    GNUNET_STATISTICS_update (stats,
                              "# empty DNS replies (usually NXDOMAIN)",
                              1,
                              GNUNET_NO);
  }
  else
  {
    record_sets++;
  }
  /* convert records to namestore import format */
  {
    struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL (rd_count)];
    unsigned int off = 0;
    char *label_utf8 = get_label_utf8 (req);

    /* convert linked list into array */
    for (rec = req->rec_head; NULL != rec; rec = rec->next)
      rd[off++] = rec->grd;
    pending_rs++;
    req->op_start_time = GNUNET_TIME_absolute_get ();
    req->qe = GNUNET_NAMESTORE_record_set_store (ns,
                                                 &req->zone->key,
                                                 label_utf8,
                                                 rd_count,
                                                 rd,
                                                 &store_completed_cb,
                                                 req);
    GNUNET_free (label_utf8);
    GNUNET_assert (NULL != req->qe);
  }
  insert_sorted (req);
  return;
}


/**
 * Process as many requests as possible from the queue.
 *
 * @param cls NULL
 */
static void
process_queue (void *cls)
{
  struct Request *req;
  unsigned int series;
  void *raw;
  size_t raw_size;
  struct GNUNET_TIME_Relative delay;

  (void) cls;
  delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc);
  idle_time = GNUNET_TIME_relative_add (idle_time, delay);
  series = 0;
  t = NULL;
  while (pending + pending_rs < THRESH)
  {
    req = GNUNET_CONTAINER_heap_peek (req_heap);
    if (NULL == req)
      break;
    if (NULL != req->qe)
      return;   /* namestore op still pending */
    if (NULL != req->rs)
    {
      GNUNET_break (0);
      return;     /* already submitted */
    }
    if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
      break;
    GNUNET_assert (req == GNUNET_CONTAINER_heap_remove_root (req_heap));
    req->hn = NULL;
    GNUNET_CONTAINER_DLL_insert (req_head, req_tail, req);
    GNUNET_assert (NULL == req->rs);
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Requesting resolution for `%s'\n",
                req->hostname);
    raw = build_dns_query (req, &raw_size);
    if (NULL == raw)
    {
      GNUNET_break (0);
      free_request (req);
      continue;
    }
    req->op_start_time = GNUNET_TIME_absolute_get ();
    req->rs = GNUNET_DNSSTUB_resolve (dns_ctx, raw, raw_size, &process_result,
                                      req);
    GNUNET_assert (NULL != req->rs);
    req->issue_num++;
    lookups++;
    pending++;
    series++;
    if (series > MAX_SERIES)
      break;
  }
  if (pending + pending_rs >= THRESH)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Stopped processing queue (%u+%u/%u)]\n",
                pending,
                pending_rs,
                THRESH);
    return;   /* wait for replies */
  }
  req = GNUNET_CONTAINER_heap_peek (req_heap);
  if (NULL == req)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Stopped processing queue: empty queue\n");
    return;
  }
  if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Waiting until %s for next record (`%s') to expire\n",
                GNUNET_STRINGS_absolute_time_to_string (req->expires),
                req->hostname);
    if (NULL != t)
      GNUNET_SCHEDULER_cancel (t);
    sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
    t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL);
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Throttling\n");
  if (NULL != t)
    GNUNET_SCHEDULER_cancel (t);
  sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
  t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, &process_queue, NULL);
}


/**
 * Iterator called during #do_shutdown() to free requests in
 * the #ns_pending map.
 *
 * @param cls NULL
 * @param key unused
 * @param value the `struct Request` to free
 * @return #GNUNET_OK
 */
static int
free_request_it (void *cls, const struct GNUNET_HashCode *key, void *value)
{
  struct Request *req = value;

  (void) cls;
  (void) key;
  free_request (req);
  return GNUNET_OK;
}


/**
 * Clean up and terminate the process.
 *
 * @param cls NULL
 */
static void
do_shutdown (void *cls)
{
  struct Request *req;
  struct Zone *zone;
  struct MissingZoneCreationCtx *mzctx;

  (void) cls;
  while (NULL != (mzctx = missing_zones_head))
  {
    GNUNET_CONTAINER_DLL_remove (missing_zones_head,
                                 missing_zones_tail,
                                 mzctx);
    if (NULL != mzctx->id_op)
    {
      GNUNET_IDENTITY_cancel (mzctx->id_op);
      mzctx->id_op = NULL;
    }
    if (NULL != mzctx->ns_qe)
    {
      GNUNET_NAMESTORE_cancel (mzctx->ns_qe);
      mzctx->ns_qe = NULL;
    }
    GNUNET_free (mzctx);
  }
  if (NULL != id)
  {
    GNUNET_IDENTITY_disconnect (id);
    id = NULL;
  }
  if (NULL != t)
  {
    GNUNET_SCHEDULER_cancel (t);
    t = NULL;
  }
  while (NULL != (req = req_head))
  {
    GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
    if (NULL != req->qe)
      GNUNET_NAMESTORE_cancel (req->qe);
    free_request (req);
  }
  while (NULL != (req = GNUNET_CONTAINER_heap_remove_root (req_heap)))
  {
    req->hn = NULL;
    if (NULL != req->qe)
      GNUNET_NAMESTORE_cancel (req->qe);
    free_request (req);
  }
  if (NULL != zone_it)
  {
    GNUNET_NAMESTORE_zone_iteration_stop (zone_it);
    zone_it = NULL;
  }
  if (NULL != ns)
  {
    GNUNET_NAMESTORE_disconnect (ns);
    ns = NULL;
  }
  if (NULL != dns_ctx)
  {
    GNUNET_DNSSTUB_stop (dns_ctx);
    dns_ctx = NULL;
  }
  if (NULL != req_heap)
  {
    GNUNET_CONTAINER_heap_destroy (req_heap);
    req_heap = NULL;
  }
  if (NULL != ns_pending)
  {
    GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &free_request_it, NULL);
    GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
    ns_pending = NULL;
  }
  while (NULL != (zone = zone_head))
  {
    GNUNET_CONTAINER_DLL_remove (zone_head, zone_tail, zone);
    GNUNET_free (zone->domain);
    GNUNET_free (zone);
  }
  if (NULL != stats)
  {
    GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
    stats = NULL;
  }
}


/**
 * Iterate over all of the zones we care about and see which records
 * we may need to re-fetch when.
 *
 * @param cls NULL
 */
static void
iterate_zones (void *cls);


/**
 * Function called if #GNUNET_NAMESTORE_records_lookup() failed.
 * Just logs an error.
 *
 * @param cls a `struct Zone`
 */
static void
ns_lookup_error_cb (void *cls)
{
  struct Zone *zone = cls;

  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Failed to load data from namestore for zone `%s'\n",
              zone->domain);
  zone_it = NULL;
  ns_iterator_trigger_next = 0;
  iterate_zones (NULL);
}


/**
 * Process a record that was stored in the namestore.
 *
 * @param cls a `struct Zone *`
 * @param key private key of the zone
 * @param label label of the records
 * @param rd_count number of entries in @a rd array, 0 if label was deleted
 * @param rd array of records with data to store
 */
static void
ns_lookup_result_cb (void *cls,
                     const struct GNUNET_CRYPTO_BlindablePrivateKey *key,
                     const char *label_utf8,
                     unsigned int rd_count,
                     const struct GNUNET_GNSRECORD_Data *rd)
{
  struct Zone *zone = cls;
  struct Request *req;
  struct GNUNET_HashCode hc;
  char *label_ace;
  char *fqdn;

  ns_iterator_trigger_next--;
  if (0 == ns_iterator_trigger_next)
  {
    ns_iterator_trigger_next = NS_BATCH_SIZE;
    GNUNET_STATISTICS_update (stats,
                              "# NAMESTORE records requested from cache",
                              ns_iterator_trigger_next,
                              GNUNET_NO);
    GNUNET_NAMESTORE_zone_iterator_next (zone_it, ns_iterator_trigger_next);
  }
  if (IDNA_SUCCESS != idna_to_ascii_8z (label_utf8,
                                        &label_ace,
                                        IDNA_ALLOW_UNASSIGNED))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _ ("Name `%s' cannot be converted to IDNA."),
                label_utf8);
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Obtained NAMESTORE reply for %s (Punycode `%s', %llu left in round\n",
              label_utf8,
              label_ace,
              (unsigned long long) ns_iterator_trigger_next);

  GNUNET_asprintf (&fqdn, "%s.%s", label_ace, zone->domain);
  GNUNET_CRYPTO_hash (fqdn, strlen (fqdn) + 1, &hc);
  GNUNET_free (fqdn);
  req = GNUNET_CONTAINER_multihashmap_get (ns_pending, &hc);
  if (NULL == req)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Ignoring record `%s' in zone `%s': not on my list!\n",
                label_ace,
                zone->domain);
    GNUNET_free (label_ace);
    return;
  }
  GNUNET_assert (GNUNET_OK ==
                 GNUNET_CONTAINER_multihashmap_remove (ns_pending, &hc, req));
  GNUNET_break (0 == GNUNET_memcmp (key, &req->zone->key));
  GNUNET_break (0 == strcasecmp (label_ace, get_label (req)));
  GNUNET_free (label_ace);
  for (unsigned int i = 0; i < rd_count; i++)
  {
    struct GNUNET_TIME_Absolute at;

    if (0 != (rd->flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
    {
      struct GNUNET_TIME_Relative rel;

      rel.rel_value_us = rd->expiration_time;
      at = GNUNET_TIME_relative_to_absolute (rel);
    }
    else
    {
      at.abs_value_us = rd->expiration_time;
    }
    add_record (req, rd->record_type, at, rd->data, rd->data_size);
  }
  if (0 == rd_count)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Empty record set in namestore for `%s'\n",
                req->hostname);
  }
  else
  {
    unsigned int pos = 0;

    cached++;
    req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
    for (struct Record *rec = req->rec_head; NULL != rec; rec = rec->next)
    {
      struct GNUNET_TIME_Absolute at;

      at.abs_value_us = rec->grd.expiration_time;
      req->expires = GNUNET_TIME_absolute_min (req->expires, at);
      pos++;
    }
    if (0 == pos)
      req->expires = GNUNET_TIME_UNIT_ZERO_ABS;
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Hot-start with %u existing records for `%s'\n",
                pos,
                req->hostname);
  }
  free_records (req);

  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Adding `%s' to worklist to start at %s\n",
              req->hostname,
              GNUNET_STRINGS_absolute_time_to_string (req->expires));
  insert_sorted (req);
}


static void
delegation_store_cont (void *cls,
                       enum GNUNET_ErrorCode ec)
{
  struct MissingZoneCreationCtx *mzctx = cls;

  mzctx->ns_qe = NULL;
  if (GNUNET_EC_NONE != ec)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to store delegation: `%s'\n",
                GNUNET_ErrorCode_get_hint (ec));
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Added delegation successfully\n");
  GNUNET_CONTAINER_DLL_remove (missing_zones_head,
                               missing_zones_tail,
                               mzctx);
  GNUNET_free (mzctx);
  if (NULL == missing_zones_head)
  {
    iterate_zones (NULL);
  }

}


static void
missing_zone_creation_cont (
  void *cls,
  const struct GNUNET_CRYPTO_BlindablePrivateKey *sk,
  enum GNUNET_ErrorCode ec)
{
  struct MissingZoneCreationCtx *mzctx = cls;
  struct GNUNET_CRYPTO_BlindablePublicKey pk;
  struct Zone *parent_zone;
  struct Zone *zone;
  const char *dot;
  const char *expected_parent_zone_name;
  char parent_delegation_label[GNUNET_DNSPARSER_MAX_NAME_LENGTH];

  if (GNUNET_EC_NONE != ec)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to create zone for `%s': %s\n",
                mzctx->req->hostname,
                GNUNET_ErrorCode_get_hint (ec));
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  mzctx->id_op = NULL;
  dot = strchr (mzctx->req->hostname, (unsigned char) '.');
  GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
              "Created missing ego `%s'\n",
              dot + 1);
  zone = GNUNET_new (struct Zone);
  zone->key = *sk;
  zone->domain = GNUNET_strdup (dot + 1);
  mzctx->req->zone = zone;
  GNUNET_CONTAINER_DLL_insert (zone_head, zone_tail, zone);

  expected_parent_zone_name = strchr (dot + 1,
                                      (unsigned char) '.');
  if (NULL == expected_parent_zone_name)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Created identity without parent zone!\n");
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  {
    int label_len = strlen (dot + 1) - strlen (expected_parent_zone_name) + 1;
    snprintf (parent_delegation_label,
              label_len,
              "%s",
              dot + 1);
    parent_delegation_label[label_len] = '\0';
  }
  for (parent_zone = zone_head;
       NULL != parent_zone;
       parent_zone = parent_zone->next)
  {
    if (0 != strcmp (expected_parent_zone_name + 1,
                     parent_zone->domain))
      continue;
    break;
  }
  GNUNET_assert (NULL != parent_zone);
  GNUNET_CRYPTO_blindable_key_get_public (sk, &pk);
  struct GNUNET_GNSRECORD_Data delegation_rd;
  char *data;
  size_t data_size;
  GNUNET_assert (GNUNET_OK ==
                 GNUNET_GNSRECORD_data_from_identity (&pk,
                                                      &data,
                                                      &data_size,
                                                      &delegation_rd.record_type
                                                      ));
  delegation_rd.flags = GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION;
  delegation_rd.expiration_time = GNUNET_TIME_UNIT_WEEKS.rel_value_us;
  delegation_rd.data = data;
  delegation_rd.data_size = data_size;
  mzctx->ns_qe = GNUNET_NAMESTORE_record_set_store (ns,
                                                    &parent_zone->key,
                                                    parent_delegation_label,
                                                    1,
                                                    &delegation_rd,
                                                    &delegation_store_cont,
                                                    mzctx);
  GNUNET_free (data);
}


/**
 * Add @a hostname to the list of requests to be made.
 *
 * @param hostname name to resolve
 */
static void
queue (const char *hostname)
{
  struct Request *req;
  const char *dot;
  struct Zone *zone;
  size_t hlen;
  struct GNUNET_HashCode hc;

  if (GNUNET_OK != GNUNET_DNSPARSER_check_name (hostname))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Refusing invalid hostname `%s'\n",
                hostname);
    rejects++;
    return;
  }
  dot = strchr (hostname, (unsigned char) '.');
  if (NULL == dot)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Refusing invalid hostname `%s' (lacks '.')\n",
                hostname);
    rejects++;
    return;
  }
  for (zone = zone_head; NULL != zone; zone = zone->next)
    if (0 == strcmp (zone->domain, dot + 1))
      break;
  hlen = strlen (hostname) + 1;
  req = GNUNET_malloc (sizeof(struct Request) + hlen);
  if (NULL == zone)
  {
    struct MissingZoneCreationCtx *mzctx;

    for (mzctx = missing_zones_head;
         NULL != mzctx;
         mzctx = mzctx->next)
    {
      const char *tmp_dot;
      tmp_dot = strchr (mzctx->req->hostname, (unsigned char) '.');
      GNUNET_assert (NULL != tmp_dot);
      if (0 != strcmp (dot + 1, tmp_dot + 1))
        continue;
      break;
    }
    if (NULL == mzctx)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                  "Domain name `%s' not in ego list! Creating.\n",
                  dot + 1);
      mzctx = GNUNET_new (struct MissingZoneCreationCtx);
      mzctx->id_op = GNUNET_IDENTITY_create (id,
                                             dot + 1,
                                             NULL,
                                             GNUNET_PUBLIC_KEY_TYPE_EDDSA,
                                             &missing_zone_creation_cont,
                                             mzctx);
      mzctx->req = req;
      GNUNET_CONTAINER_DLL_insert (missing_zones_head,
                                   missing_zones_tail,
                                   mzctx);
    }
  }
  else
  {
    req->zone = zone;
  }

  req->hostname = (char *) &req[1];
  GNUNET_memcpy (req->hostname, hostname, hlen);
  req->id = (uint16_t) GNUNET_CRYPTO_random_u32 (UINT16_MAX);
  GNUNET_CRYPTO_hash (req->hostname, hlen, &hc);
  if (GNUNET_OK != GNUNET_CONTAINER_multihashmap_put (
        ns_pending,
        &hc,
        req,
        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                "Duplicate hostname `%s' ignored\n",
                hostname);
    GNUNET_free (req);
    return;
  }
}


/**
 * We have completed the initial iteration over the namestore's database.
 * This function is called on each of the remaining records in
 * #move_to_queue to #queue() them, as we will simply not find existing
 * records for them any longer.
 *
 * @param cls NULL
 * @param key unused
 * @param value a `struct Request`
 * @return #GNUNET_OK (continue to iterate)
 */
static int
move_to_queue (void *cls, const struct GNUNET_HashCode *key, void *value)
{
  struct Request *req = value;

  (void) cls;
  (void) key;
  insert_sorted (req);
  return GNUNET_OK;
}


/**
 * Iterate over all of the zones we care about and see which records
 * we may need to re-fetch when.
 *
 * @param cls NULL
 */
static void
iterate_zones (void *cls)
{
  static struct Zone *last;

  (void) cls;
  if (NULL != zone_it)
  {
    zone_it = NULL;
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Finished iteration over zone `%s'!\n",
                last->domain);
    /* subtract left-overs from previous iteration */
    GNUNET_STATISTICS_update (stats,
                              "# NAMESTORE records requested from cache",
                              (long long) (-ns_iterator_trigger_next),
                              GNUNET_NO);
    ns_iterator_trigger_next = 0;
  }
  GNUNET_assert (NULL != zone_tail);
  if (zone_tail == last)
  {
    /* Done iterating over relevant zones in NAMESTORE, move
       rest of hash map to work queue as well. */
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Finished all NAMESTORE iterations!\n");
    GNUNET_STATISTICS_set (stats,
                           "# Domain names without cached reply",
                           GNUNET_CONTAINER_multihashmap_size (ns_pending),
                           GNUNET_NO);
    GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &move_to_queue, NULL);
    GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
    ns_pending = NULL;
    start_time_reg_proc = GNUNET_TIME_absolute_get ();
    total_reg_proc_dns = 0;
    total_reg_proc_dns_ns = 0;
    return;
  }
  if (NULL == last)
    last = zone_head;
  else
    last = last->next;
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Starting iteration over zone `%s'!\n",
              last->domain);
  /* subtract left-overs from previous iteration */
  GNUNET_STATISTICS_update (stats,
                            "# NAMESTORE records requested from cache",
                            1,
                            GNUNET_NO);
  ns_iterator_trigger_next = 1;
  GNUNET_STATISTICS_update (stats, "# zones iterated", 1, GNUNET_NO);
  zone_it = GNUNET_NAMESTORE_zone_iteration_start (ns,
                                                   &last->key,
                                                   &ns_lookup_error_cb,
                                                   NULL,
                                                   &ns_lookup_result_cb,
                                                   last,
                                                   &iterate_zones,
                                                   NULL);
}


/**
 * Begin processing hostnames from stdin.
 *
 * @param cls NULL
 */
static void
process_stdin (void *cls)
{
  static struct GNUNET_TIME_Absolute last;
  char hn[256];
  char *hn_utf8;

  (void) cls;
  t = NULL;
  while (NULL != fgets (hn, sizeof(hn), stdin))
  {
    if (strlen (hn) > 0)
      hn[strlen (hn) - 1] = '\0';  /* eat newline */
    if (0 == stdin_lines_read)
      last = GNUNET_TIME_absolute_get ();
    stdin_lines_read++;
    if (0 == stdin_lines_read % 100000)
    {
      struct GNUNET_TIME_Relative delta;

      delta = GNUNET_TIME_absolute_get_duration (last);
      last = GNUNET_TIME_absolute_get ();
      fprintf (stderr,
               "Read 100000 domain names in %s\n",
               GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES));
      GNUNET_STATISTICS_set (stats, "# domain names provided", stdin_lines_read,
                             GNUNET_NO);
    }
    idna_to_unicode_8z8z (hn,
                          &hn_utf8,
                          IDNA_ALLOW_UNASSIGNED);
    queue (hn_utf8);
    GNUNET_free (hn_utf8);
  }
  fprintf (stderr,
           "Done reading %llu domain names\n",
           (unsigned long long) stdin_lines_read);
  GNUNET_STATISTICS_set (stats, "# domain names provided", stdin_lines_read,
                         GNUNET_NO);
  // Only start iteration immediately if no zones are missing.
  if (NULL == missing_zones_head)
  {
    iterate_zones (NULL);
  }
}


/**
 * Method called to inform about the egos of this peer.
 *
 * When used with #GNUNET_IDENTITY_connect, this function is
 * initially called for all egos and then again whenever a
 * ego's name changes or if it is deleted.  At the end of
 * the initial pass over all egos, the function is once called
 * with 'NULL' for @a ego. That does NOT mean that the callback won't
 * be invoked in the future or that there was an error.
 *
 * When used with #GNUNET_IDENTITY_create, this
 * function is only called ONCE, and 'NULL' being passed in @a ego does
 * indicate an error (for example because name is taken or no default value is
 * known).  If @a ego is non-NULL and if '*ctx' is set in those callbacks, the
 * value WILL be passed to a subsequent call to the identity callback of
 * #GNUNET_IDENTITY_connect (if that one was not NULL).
 *
 * When an identity is renamed, this function is called with the
 * (known) @a ego but the NEW @a name.
 *
 * When an identity is deleted, this function is called with the
 * (known) ego and "NULL" for the @a name.  In this case,
 * the @a ego is henceforth invalid (and the @a ctx should also be
 * cleaned up).
 *
 * @param cls closure
 * @param ego ego handle, NULL for end of list
 * @param ctx context for application to store data for this ego
 *                 (during the lifetime of this process, initially NULL)
 * @param name name assigned by the user for this ego,
 *                   NULL if the user just deleted the ego and it
 *                   must thus no longer be used
 */
static void
identity_cb (void *cls,
             struct GNUNET_IDENTITY_Ego *ego,
             void **ctx,
             const char *name)
{
  (void) cls;
  (void) ctx;
  static int initial_iteration = GNUNET_YES;

  if (GNUNET_NO == initial_iteration)
    return;
  if (NULL == ego)
  {
    /* end of iteration */
    if (NULL == zone_head)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "No zone found\n");
      GNUNET_SCHEDULER_shutdown ();
      return;
    }
    initial_iteration = GNUNET_NO;
    /* zone_head non-null, process hostnames from stdin */
    t = GNUNET_SCHEDULER_add_now (&process_stdin, NULL);
    return;
  }
  if (NULL != name)
  {
    struct Zone *zone;

    zone = GNUNET_new (struct Zone);
    zone->key = *GNUNET_IDENTITY_ego_get_private_key (ego);
    zone->domain = GNUNET_strdup (name);
    GNUNET_CONTAINER_DLL_insert (zone_head, zone_tail, zone);
  }
}


/**
 * Process requests from the queue, then if the queue is
 * not empty, try again.
 *
 * @param cls NULL
 * @param args remaining command-line arguments
 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
 * @param cfg configuration
 */
static void
run (void *cls,
     char *const *args,
     const char *cfgfile,
     const struct GNUNET_CONFIGURATION_Handle *cfg)
{
  (void) cls;
  (void) args;
  (void) cfgfile;
  stats = GNUNET_STATISTICS_create ("zoneimport", cfg);
  req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
  ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size, GNUNET_NO);
  if (NULL == ns_pending)
  {
    fprintf (stderr, "Failed to allocate memory for main hash map\n");
    return;
  }
  dns_ctx = GNUNET_DNSSTUB_start (256);
  if (NULL == dns_ctx)
  {
    fprintf (stderr, "Failed to initialize GNUnet DNS STUB\n");
    return;
  }
  if (NULL == args[0])
  {
    fprintf (stderr,
             "You must provide a list of DNS resolvers on the command line\n");
    return;
  }
  for (unsigned int i = 0; NULL != args[i]; i++)
  {
    if (GNUNET_OK != GNUNET_DNSSTUB_add_dns_ip (dns_ctx, args[i]))
    {
      fprintf (stderr, "Failed to use `%s' for DNS resolver\n", args[i]);
      return;
    }
  }


  GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
  ns = GNUNET_NAMESTORE_connect (cfg);
  if (NULL == ns)
  {
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  id = GNUNET_IDENTITY_connect (cfg, &identity_cb, NULL);
}


/**
 * Call with IP address of resolver to query.
 *
 * @param argc should be 2
 * @param argv[1] should contain IP address
 * @return 0 on success
 */
int
main (int argc, char *const *argv)
{
  struct GNUNET_GETOPT_CommandLineOption options[] =
  { GNUNET_GETOPT_option_uint ('s',
                               "size",
                               "MAPSIZE",
                               gettext_noop (
                                 "size to use for the main hash map"),
                               &map_size),
    GNUNET_GETOPT_option_relative_time (
      'm',
      "minimum-expiration",
      "RELATIVETIME",
      gettext_noop ("minimum expiration time we assume for imported records"),
      &minimum_expiration_time),
    GNUNET_GETOPT_OPTION_END };
  int ret;

  if (GNUNET_OK !=
      (ret = GNUNET_PROGRAM_run (GNUNET_OS_project_data_gnunet (),
                                 argc,
                                 argv,
                                 "gnunet-zoneimport",
                                 "import DNS zone into namestore",
                                 options,
                                 &run,
                                 NULL)))
    return 1;
  fprintf (stderr,
           "Rejected %u names, had %u cached, did %u lookups, stored %u record sets\n"
           "Found %u records, %u lookups failed, %u/%u pending on shutdown\n",
           rejects,
           cached,
           lookups,
           record_sets,
           records,
           failures,
           pending,
           pending_rs);
  return 0;
}


/* end of gnunet-zoneimport.c */
