/*
 * Copyright (C) 2004, 2005 Jean-Yves Lefort
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of Jean-Yves Lefort nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "config.h"
#include <string.h>
#include <stdarg.h>
#include <stdlib.h>
#include <glib/gi18n-lib.h>
#include <libsoup/soup.h>
#include <libsoup/soup-message-filter.h>
#include <libxml/HTMLparser.h>
#include "translate.h"
#include "translate-generic-service.h"
#include "translate-generic-main.h"
#include "translate-generic-parser.h"
#include "translate-generic-soup-cookie-jar.h"

#define MAKE_WARNING_PREFIX(service, group_pos, attribute, element) \
  g_strdup_printf(_("in %s, group %i, \"%s\" attribute of \"%s\" element"), \
		  translate_service_get_name((service)), \
		  (group_pos), (attribute), (element))

enum {
  PROP_0,
  PROP_GROUPS
};

struct _TranslateGenericServicePrivate
{
  GSList *groups;
};

typedef struct
{
  GSList			**pairs;
  TranslatePairFlags		flags;
} GetPairsInfo;

typedef struct
{
  gboolean			found;
  const char			*from;
  const char			*to;
} GetGroupInfo;

typedef enum
{
  TRANSFER_FOLLOW_REFRESH	= 1 << 0,
  TRANSFER_CONVERT		= 1 << 1
} TransferFlags;

typedef enum
{
  HTML_CONTEXT_PRE_HEAD,
  HTML_CONTEXT_HEAD,
  HTML_CONTEXT_POST_HEAD
} HTMLContext;

typedef struct
{
  SoupSession			*session;
  TranslateProgressFunc		progress_func;
  gpointer			user_data;
  unsigned int			length;
  unsigned int			received;

  /* HTML parser */
  gboolean			parse_html;
  HTMLContext			html_context;
  GHashTable			*html_http_equiv;
} TransferInfo;

static GObjectClass *parent_class = NULL;

static void translate_generic_service_register_type (GType *type);
static void translate_generic_service_class_init (TranslateGenericServiceClass *class);
static void translate_generic_service_init (TranslateGenericService *service);
static void translate_generic_service_finalize (GObject *object);
static void translate_generic_service_set_property (GObject *object,
						    unsigned int prop_id,
						    const GValue *value,
						    GParamSpec *pspec);

static gboolean translate_generic_service_get_pairs (TranslateService *service,
						     GSList **pairs,
						     TranslateProgressFunc progress_func,
						     gpointer user_data,
						     GError **err);
static gboolean translate_generic_service_get_pairs_cb (const char *from,
							const char *to,
							gpointer user_data);

static TranslateGenericGroup *translate_generic_service_get_group (TranslateGenericService *service,
								   const char *from,
								   const char *to,
								   int *pos);
static gboolean translate_generic_service_get_group_cb (const char *from,
							const char *to,
							gpointer user_data);

static char *translate_generic_service_get (const char *uri,
					    const char *post,
					    const char *post_content_type,
					    const GSList *headers,
					    TransferFlags flags,
					    TranslateProgressFunc progress_func,
					    gpointer user_data,
					    GError **err);

static const char *translate_generic_service_get_header (SoupMessage *message,
							 TransferInfo *info,
							 const char *name);

static void translate_generic_service_log_connect (SoupMessage *message);
static void translate_generic_service_log_wrote_headers_h (SoupMessage *message,
							   gpointer user_data);
static void translate_generic_service_log_wrote_body_h (SoupMessage *message,
							gpointer user_data);
static void translate_generic_service_log_got_headers_h (SoupMessage *message,
							 gpointer user_data);
static void translate_generic_service_log_got_body_h (SoupMessage *message,
						      gpointer user_data);
static void translate_generic_service_log_headers_cb (const char *key,
						      const char *value,
						      gpointer user_data);

static void translate_generic_service_progress_got_headers_h (SoupMessage *message,
							      gpointer user_data);
static void translate_generic_service_progress_got_chunk_h (SoupMessage *message,
							    gpointer user_data);

static void translate_generic_service_html_got_headers_h (SoupMessage *message,
							  gpointer user_data);
static void translate_generic_service_html_got_body_h (SoupMessage *message,
						       gpointer user_data);
static void translate_generic_service_html_start_element_cb (gpointer user_data,
							     const xmlChar *name,
							     const xmlChar **atts);
static void translate_generic_service_html_end_element_cb (gpointer user_data,
							   const xmlChar *name);

static void translate_generic_service_refresh_got_body_h (SoupMessage *message,
							  gpointer user_data);

static void translate_generic_service_redirect_handler (SoupMessage *message,
							gpointer user_data);

static char *translate_generic_service_translate_text (TranslateService *service,
						       const char *text,
						       const char *from,
						       const char *to,
						       TranslateProgressFunc progress_func,
						       gpointer user_data,
						       GError **err);

char *translate_generic_service_expand (const char *warning_prefix,
					const char *str,
					...);
static char *translate_generic_service_expand_variable (const char *warning_prefix,
							const char *variable,
							GHashTable *subs);
static char *translate_generic_service_modify_value (const char *warning_prefix,
						     const char *value,
						     const char *modifier_name,
						     const char *modifier_value);

static char *translate_generic_service_translate_web_page (TranslateService *service,
							   const char *url,
							   const char *from,
							   const char *to,
							   TranslateProgressFunc progress_func,
							   gpointer user_data,
							   GError **err);

static SoupSession *translate_generic_service_soup_session_sync_new (void);

GType
translate_generic_service_get_type (void)
{
  static GType type;
  static GOnce once = G_ONCE_INIT;

  g_once(&once, (GThreadFunc) translate_generic_service_register_type, &type);

  return type;
}

static void
translate_generic_service_register_type (GType *type)
{
  static const GTypeInfo info = {
    sizeof(TranslateGenericServiceClass),
    NULL,
    NULL,
    (GClassInitFunc) translate_generic_service_class_init,
    NULL,
    NULL,
    sizeof(TranslateGenericService),
    0,
    (GInstanceInitFunc) translate_generic_service_init
  };
  
  *type = g_type_register_static(TRANSLATE_TYPE_SERVICE,
				 "TranslateGenericService",
				 &info,
				 0);
}

static void
translate_generic_service_class_init (TranslateGenericServiceClass *class)
{
  GObjectClass *object_class = G_OBJECT_CLASS(class);
  TranslateServiceClass *service_class = TRANSLATE_SERVICE_CLASS(class);
  
  g_type_class_add_private(class, sizeof(TranslateGenericServicePrivate));
  parent_class = g_type_class_peek_parent(class);

  object_class->finalize = translate_generic_service_finalize;
  object_class->set_property = translate_generic_service_set_property;

  service_class->get_pairs = translate_generic_service_get_pairs;
  service_class->translate_text = translate_generic_service_translate_text;
  service_class->translate_web_page = translate_generic_service_translate_web_page;

  g_object_class_install_property(object_class,
				  PROP_GROUPS,
				  g_param_spec_pointer("groups",
						       NULL,
						       NULL,
						       G_PARAM_WRITABLE | G_PARAM_CONSTRUCT_ONLY));
}

static void
translate_generic_service_init (TranslateGenericService *service)
{
  service->priv = G_TYPE_INSTANCE_GET_PRIVATE(service,
					      TRANSLATE_GENERIC_TYPE_SERVICE,
					      TranslateGenericServicePrivate);
}

static void
translate_generic_service_finalize (GObject *object)
{
  TranslateGenericService *service = TRANSLATE_GENERIC_SERVICE(object);

  g_slist_foreach(service->priv->groups, (GFunc) translate_generic_group_unref, NULL);
  g_slist_free(service->priv->groups);

  parent_class->finalize(object);
}

static void
translate_generic_service_set_property (GObject *object,
					unsigned int prop_id,
					const GValue *value,
					GParamSpec *pspec)
{
  TranslateGenericService *service = TRANSLATE_GENERIC_SERVICE(object);

  switch (prop_id)
    {
    case PROP_GROUPS:
      service->priv->groups = g_slist_copy(g_value_get_pointer(value));
      g_slist_foreach(service->priv->groups, (GFunc) translate_generic_group_ref, NULL);
      break;

    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
      break;
    }
}

static gboolean
translate_generic_service_get_pairs (TranslateService *service,
				     GSList **pairs,
				     TranslateProgressFunc progress_func,
				     gpointer user_data,
				     GError **err)
{
  TranslateGenericService *generic = TRANSLATE_GENERIC_SERVICE(service);
  GetPairsInfo info;
  GSList *l;

  info.pairs = pairs;
  *info.pairs = NULL;

  for (l = generic->priv->groups; l != NULL; l = l ->next)
    {
      TranslateGenericGroup *group = l->data;
      
      info.flags = 0;
      if (group->text_location)
	info.flags |= TRANSLATE_PAIR_TEXT;
      if (group->web_page_location)
	info.flags |= TRANSLATE_PAIR_WEB_PAGE;

      translate_generic_group_foreach_pair(group, translate_generic_service_get_pairs_cb, &info);
    }

  return TRUE;
}

static gboolean
translate_generic_service_get_pairs_cb (const char *from,
					const char *to,
					gpointer user_data)
{
  GetPairsInfo *info = user_data;

  *info->pairs = g_slist_append(*info->pairs, translate_pair_new(info->flags, from, to));

  return TRUE;			/* continue */
}

static TranslateGenericGroup *
translate_generic_service_get_group (TranslateGenericService *service,
				     const char *from,
				     const char *to,
				     int *pos)
{
  GSList *l;
  int _pos;
  GetGroupInfo info = { FALSE, from, to };

  g_return_val_if_fail(TRANSLATE_GENERIC_IS_SERVICE(service), NULL);
  g_return_val_if_fail(from != NULL, NULL);
  g_return_val_if_fail(to != NULL, NULL);
  g_return_val_if_fail(pos != NULL, NULL);

  for (l = service->priv->groups, _pos = 1; l != NULL; l = l ->next, _pos++)
    {
      TranslateGenericGroup *group = l->data;

      translate_generic_group_foreach_pair(group, translate_generic_service_get_group_cb, &info);
      if (info.found)
	{
	  *pos = _pos;
	  return group;
	}
    }

  *pos = -1;
  return NULL;
}

static gboolean
translate_generic_service_get_group_cb (const char *from,
					const char *to,
					gpointer user_data)
{
  GetGroupInfo *info = user_data;

  if (! g_ascii_strcasecmp(from, info->from) && ! g_ascii_strcasecmp(to, info->to))
    {
      info->found = TRUE;
      return FALSE;		/* abort */
    }
  else
    return TRUE;		/* continue */
}

static char *
translate_generic_service_get (const char *uri,
			       const char *post,
			       const char *post_content_type,
			       const GSList *headers,
			       TransferFlags flags,
			       TranslateProgressFunc progress_func,
			       gpointer user_data,
			       GError **err)
{
  TransferInfo info;
  SoupMessage *message;
  const GSList *l;
  char *response = NULL;

  g_return_val_if_fail(uri != NULL, FALSE);

  message = soup_message_new(post ? SOUP_METHOD_POST : SOUP_METHOD_GET, uri);
  if (! message)
    {
      g_set_error(err,
		  TRANSLATE_GENERIC_SERVICE_ERROR,
		  TRANSLATE_GENERIC_SERVICE_ERROR_TRANSFER,
		  _("unable to parse URI \"%s\""), uri);
      return NULL;
    }

  if (post)
    {
      g_return_val_if_fail(post_content_type != NULL, NULL);
      soup_message_set_request(message,
			       post_content_type,
			       SOUP_BUFFER_USER_OWNED,
			       (char *) post,
			       strlen(post));
    }
  
  for (l = headers; l != NULL; l = l->next)
    {
      TranslateGenericHttpHeader *header = l->data;
      soup_message_add_header(message->request_headers, header->name, header->value);
    }
  
  info.session = translate_generic_service_soup_session_sync_new();
  info.parse_html = FALSE;
  info.html_http_equiv = NULL;

  if (translate_generic_debug_flags & TRANSLATE_GENERIC_DEBUG_LOG_TRANSFERS)
    g_object_connect(message,
		     "signal::wrote-headers", translate_generic_service_log_wrote_headers_h, &info,
		     "signal::wrote-body", translate_generic_service_log_wrote_body_h, &info,
		     "signal::got-headers", translate_generic_service_log_got_headers_h, &info,
		     "signal::got-body", translate_generic_service_log_got_body_h, &info,
		     NULL);

  if (progress_func)
    {
      info.progress_func = progress_func;
      info.user_data = user_data;
      info.length = -1;
      info.received = 0;
      
      g_object_connect(message,
		       "signal::got-headers", translate_generic_service_progress_got_headers_h, &info,
		       "signal::got-chunk", translate_generic_service_progress_got_chunk_h, &info,
		       NULL);
    }

  /*
   * We parse the HTML to retrieve the http-equiv meta tags, and we
   * only need them if we convert or follow refresh.
   */
  if ((flags & TRANSFER_FOLLOW_REFRESH) || (flags & TRANSFER_CONVERT))
    g_object_connect(message,
		     "signal::got-headers", translate_generic_service_html_got_headers_h, &info,
		     "signal::got-body", translate_generic_service_html_got_body_h, &info,
		     NULL);

  if (flags & TRANSFER_FOLLOW_REFRESH)
    g_signal_connect(message, "got-body", G_CALLBACK(translate_generic_service_refresh_got_body_h), &info);

  /* http://bugzilla.ximian.com/show_bug.cgi?id=70688 */
  soup_message_set_flags(message, SOUP_MESSAGE_NO_REDIRECT);
  soup_message_add_status_class_handler(message,
					SOUP_STATUS_CLASS_REDIRECT,
					SOUP_HANDLER_POST_BODY,
					translate_generic_service_redirect_handler,
					info.session);

  if (translate_generic_debug_flags & TRANSLATE_GENERIC_DEBUG_LOG_TRANSFERS)
    translate_generic_service_log_connect(message);

  soup_session_send_message(info.session, message);
  g_object_unref(info.session);

  if (SOUP_STATUS_IS_SUCCESSFUL(message->status_code))
    {
      const char *charset = NULL;
      
      if (flags & TRANSFER_CONVERT)
	{
	  const char *content_type;

	  content_type = translate_generic_service_get_header(message, &info, "Content-Type");
	  if (content_type)
	    {
	      charset = translate_ascii_strcasestr(content_type, "charset=");
	      if (charset)
		charset += 8;
	    }
	}

      if (charset)
	response = g_convert(message->response.body, message->response.length, "UTF-8", charset, NULL, NULL, err);
      else
	{
	  if ((flags & TRANSFER_CONVERT) && ! g_utf8_validate(message->response.body, message->response.length, NULL))
	    g_set_error(err,
			TRANSLATE_GENERIC_SERVICE_ERROR,
			TRANSLATE_GENERIC_SERVICE_ERROR_TRANSFER,
			_("invalid UTF-8"));
	  else
	    response = g_strndup(message->response.body, message->response.length);
	}
    }
  else
    {
      if (message->status_code == SOUP_STATUS_CANCELLED)
	g_set_error(err,
		    TRANSLATE_ERROR,
		    TRANSLATE_ERROR_CANCELLED,
		    "%s", message->reason_phrase);
      else
	g_set_error(err,
		    TRANSLATE_GENERIC_SERVICE_ERROR,
		    TRANSLATE_GENERIC_SERVICE_ERROR_TRANSFER,
		    "%s", message->reason_phrase);
    }

  if (info.html_http_equiv)
    g_hash_table_destroy(info.html_http_equiv);

  g_object_unref(message);

  return response;
}

static const char *
translate_generic_service_get_header (SoupMessage *message,
				      TransferInfo *info,
				      const char *name)
{
  const char *value;

  g_return_val_if_fail(SOUP_IS_MESSAGE(message), NULL);
  g_return_val_if_fail(info != NULL, NULL);
  g_return_val_if_fail(name != NULL, NULL);

  value = info->html_http_equiv
    ? g_hash_table_lookup(info->html_http_equiv, name)
    : NULL;

  if (! value)
    value = soup_message_get_header(message->response_headers, name);

  return value;
}

static void
translate_generic_service_log_connect (SoupMessage *message)
{
  const SoupUri *uri;

  uri = soup_message_get_uri(message);
  g_log(G_LOG_DOMAIN, G_LOG_LEVEL_DEBUG, _("connecting to %s:%i"), uri->host, uri->port);
}

static void
translate_generic_service_log_wrote_headers_h (SoupMessage *message,
					       gpointer user_data)
{
  const SoupUri *suri;
  char *uri;

  suri = soup_message_get_uri(message);
  uri = soup_uri_to_string(suri, FALSE);

  g_log(G_LOG_DOMAIN, G_LOG_LEVEL_DEBUG, "> %s %s", message->method, uri);
  g_free(uri);

  soup_message_foreach_header(message->request_headers, (GHFunc) translate_generic_service_log_headers_cb, ">");
}

static void
translate_generic_service_log_wrote_body_h (SoupMessage *message,
					    gpointer user_data)
{
  if (message->request.body)
    g_log(G_LOG_DOMAIN, G_LOG_LEVEL_DEBUG, "> %s", message->request.body);
}

static void
translate_generic_service_log_got_headers_h (SoupMessage *message,
					     gpointer user_data)
{
  g_log(G_LOG_DOMAIN, G_LOG_LEVEL_DEBUG, "< %s", soup_status_get_phrase(message->status_code));

  soup_message_foreach_header(message->response_headers, (GHFunc) translate_generic_service_log_headers_cb, "<");

  if (message->response.body)
    g_log(G_LOG_DOMAIN, G_LOG_LEVEL_DEBUG, "< %s", message->response.body);
}

static void
translate_generic_service_log_got_body_h (SoupMessage *message,
					  gpointer user_data)
{
  if (message->response.body)
    g_log(G_LOG_DOMAIN, G_LOG_LEVEL_DEBUG, "< %s", message->response.body);
}

static void
translate_generic_service_log_headers_cb (const char *key,
					  const char *value,
					  gpointer user_data)
{
  const char *prefix = user_data;

  g_log(G_LOG_DOMAIN, G_LOG_LEVEL_DEBUG, "%s %s: %s", prefix, key, value);
}

static void
translate_generic_service_progress_got_headers_h (SoupMessage *message,
						  gpointer user_data)
{
  TransferInfo *info = user_data;
  const char *content_length;

  content_length = soup_message_get_header(message->response_headers, "Content-Length");
  info->length = (content_length
		  && *content_length
		  && strspn(content_length, "0123456789") == strlen(content_length))
    ? atoi(content_length) : -1;
  info->received = 0;
}

static void
translate_generic_service_progress_got_chunk_h (SoupMessage *message,
						gpointer user_data)
{
  TransferInfo *info = user_data;
  double progress;

  if (info->length == -1)
    progress = -1;
  else
    {
      info->received += message->response.length;
      progress = (double) info->received / info->length;
      progress = CLAMP(progress, 0.0, 1.0);
    }
  
  if (! info->progress_func(progress, info->user_data))
    soup_session_abort(info->session);
}

static void
translate_generic_service_html_got_headers_h (SoupMessage *message,
					      gpointer user_data)
{
  TransferInfo *info = user_data;
  const char *content_type;

  content_type = soup_message_get_header(message->response_headers, "Content-Type");
  info->parse_html = content_type
    && (g_str_has_prefix(content_type, "text/html")
	|| g_str_has_prefix(content_type, "application/xhtml+xml")
	|| g_str_has_prefix(content_type, "application/xml")
	|| g_str_has_prefix(content_type, "text/xml"));
}

static void
translate_generic_service_html_got_body_h (SoupMessage *message,
					   gpointer user_data)
{
  TransferInfo *info = user_data;

  if (info->html_http_equiv)
    {
      g_hash_table_destroy(info->html_http_equiv);
      info->html_http_equiv = NULL;
    }

  if (info->parse_html && message->response.length > 0)
    {
      char *body;
      xmlSAXHandler sax_handler = { NULL };

      info->html_context = HTML_CONTEXT_PRE_HEAD;
      info->html_http_equiv = g_hash_table_new_full(translate_ascii_strcase_hash,
						    translate_ascii_strcase_equal,
						    g_free,
						    g_free);

      sax_handler.startElement = translate_generic_service_html_start_element_cb;
      sax_handler.endElement = translate_generic_service_html_end_element_cb;

      body = g_strndup(message->response.body, message->response.length);
      htmlSAXParseDoc(body, NULL, &sax_handler, user_data);
      g_free(body);
    }
}

static void
translate_generic_service_html_start_element_cb (gpointer user_data,
						 const xmlChar *name,
						 const xmlChar **atts)
{
  TransferInfo *info = user_data;

  if (info->html_context == HTML_CONTEXT_PRE_HEAD)
    {
      if (! g_ascii_strcasecmp(name, "head"))
	info->html_context = HTML_CONTEXT_HEAD;
    }
  else if (info->html_context == HTML_CONTEXT_HEAD)
    {
      if (! g_ascii_strcasecmp(name, "meta"))
	{
	  int i;
	  const char *http_equiv = NULL;
	  
	  for (i = 0; atts[i] && atts[i + 1]; i += 2)
	    if (! g_ascii_strcasecmp(atts[i], "http-equiv"))
	      {
		http_equiv = atts[i + 1];
		break;
	      }

	  if (http_equiv)
	    {
	      const char *content = NULL;

	      for (i = 0; atts[i] && atts[i + 1]; i += 2)
		if (! g_ascii_strcasecmp(atts[i], "content"))
		  {
		    content = atts[i + 1];
		    break;
		  }

	      if (content)
		g_hash_table_insert(info->html_http_equiv, g_strdup(http_equiv), g_strdup(content));
	    }
	}
    }
}

static void
translate_generic_service_html_end_element_cb (gpointer user_data,
					       const xmlChar *name)
{
  TransferInfo *info = user_data;

  if (info->html_context == HTML_CONTEXT_HEAD
      && ! g_ascii_strcasecmp(name, "head"))
    info->html_context = HTML_CONTEXT_POST_HEAD;
}

static void
translate_generic_service_refresh_got_body_h (SoupMessage *message,
					      gpointer user_data)
{
  TransferInfo *info = user_data;
  const char *refresh_uri;
  SoupUri *new_uri = NULL;

  refresh_uri = translate_generic_service_get_header(message, info, "Refresh");
  if (refresh_uri)
    {
      refresh_uri = translate_ascii_strcasestr(refresh_uri, "url=");
      if (refresh_uri)
	refresh_uri += 4;
    }
  
  if (refresh_uri)
    {
      new_uri = soup_uri_new(refresh_uri);
      if (! new_uri)
	{
	  const SoupUri *base_uri;
	  
	  base_uri = soup_message_get_uri(message);
	  new_uri = soup_uri_new_with_base(base_uri, refresh_uri);
	}
    }
  
  if (new_uri)
    {
      soup_message_set_uri(message, new_uri);
      soup_uri_free(new_uri);
      
      if (translate_generic_debug_flags & TRANSLATE_GENERIC_DEBUG_LOG_TRANSFERS)
	translate_generic_service_log_connect(message);

      soup_session_requeue_message(info->session, message);
    }
}

static void
translate_generic_service_redirect_handler (SoupMessage *message,
					    gpointer user_data)
{
  const char *new_location;

  new_location = soup_message_get_header(message->response_headers, "Location");
  if (new_location)
    {
      SoupSession *session = user_data;
      SoupUri *new_uri;
      
      new_uri = soup_uri_new(new_location);
      if (! new_uri)
	{
	  const SoupUri *base_uri;

	  base_uri = soup_message_get_uri(message);
	  new_uri = soup_uri_new_with_base(base_uri, new_location);

	  if (! new_uri)
	    {
	      soup_message_set_status_full(message, SOUP_STATUS_MALFORMED, _("invalid redirect URL"));
	      return;
	    }
	}

      soup_message_set_uri(message, new_uri);
      soup_uri_free(new_uri);

      if (translate_generic_debug_flags & TRANSLATE_GENERIC_DEBUG_LOG_TRANSFERS)
	translate_generic_service_log_connect(message);

      soup_session_requeue_message(session, message);
    }
}

static char *
translate_generic_service_translate_text (TranslateService *service,
					  const char *text,
					  const char *from,
					  const char *to,
					  TranslateProgressFunc progress_func,
					  gpointer user_data,
					  GError **err)
{
  TranslateGenericService *generic = TRANSLATE_GENERIC_SERVICE(service);
  TranslateGenericGroup *group;
  int group_pos;
  const char *service_from;
  const char *service_to;
  char *warning_prefix;
  char *url;
  char *post = NULL;
  GSList *headers;
  char *response;
  GString *answer = NULL;

  group = translate_generic_service_get_group(generic, from, to, &group_pos);
  g_return_val_if_fail(group != NULL, NULL);

  service_from = translate_generic_group_get_service_tag(group, from);
  service_to = translate_generic_group_get_service_tag(group, to);

  warning_prefix = MAKE_WARNING_PREFIX(service, group_pos, "url", "text-translation");
  url = translate_generic_service_expand(warning_prefix,
					 group->text_location->url,
					 "text", text,
					 "from", service_from,
					 "to", service_to,
					 NULL);
  g_free(warning_prefix);

  if (group->text_location->post)
    {
      warning_prefix = MAKE_WARNING_PREFIX(service, group_pos, "post", "text-translation");
      post = translate_generic_service_expand(warning_prefix,
					      group->text_location->post,
					      "text", text,
					      "from", service_from,
					      "to", service_to,
					      NULL);
      g_free(warning_prefix);
    }
  
  headers = g_slist_copy(group->http_headers);
  headers = g_slist_concat(headers, g_slist_copy(group->text_location->http_headers));
    
  response = translate_generic_service_get(url,
					   post,
					   group->text_location->content_type,
					   headers,
					   TRANSFER_FOLLOW_REFRESH | TRANSFER_CONVERT,
					   progress_func,
					   user_data,
					   err);

  g_free(url);
  g_free(post);
  g_slist_free(headers);

  if (response)
    {
      const char *work = response;
      char *error = NULL;
      GSList *l;

      for (l = group->text_error_markers; l != NULL && ! error; l = l->next)
	{
	  const char *marker = l->data;

	  error = strstr(work, marker);
	}

      if (error)
	g_set_error(err,
		    TRANSLATE_GENERIC_SERVICE_ERROR,
		    TRANSLATE_GENERIC_SERVICE_ERROR_FAILED,
		    _("server failure"));
      else
	{
	  char *raw = NULL;
	  
	  for (l = group->text_pre_markers; l != NULL && work; l = l->next)
	    {
	      const char *marker = l->data;

	      work = strstr(work, marker);
	      if (work)
		work += strlen(marker);
	    }
      
	  if (work)
	    {
	      if (group->text_post_marker)
		{
		  char *s;
		  
		  s = strstr(work, group->text_post_marker);
		  if (s)
		    raw = g_strndup(work, s - work);
		}
	      else
		raw = g_strdup(response);
	    }

	  if (raw)
	    {
	      char *expanded;
	      const char *s;
	      int len;

	      expanded = translate_sgml_ref_expand(raw);
	      g_free(raw);

	      /*
	       * If the service has removed the leading and/or
	       * trailing whitespace, restore it.
	       */

	      answer = g_string_new(NULL);

	      for (s = text;
		   *s && g_unichar_isspace(g_utf8_get_char(s));
		   s = g_utf8_next_char(s));

	      len = s - text;
	      if (len > 0 && strncmp(expanded, text, len))
		g_string_append_len(answer, text, len);

	      g_string_append(answer, expanded);

	      if (*s)
		{ /* there was a middle block, handle trailing spaces */
		  for (s = g_utf8_find_prev_char(text, strchr(text, 0));
		       s && g_unichar_isspace(g_utf8_get_char(s));
		       s = g_utf8_find_prev_char(text, s));
		  s = s ? g_utf8_next_char(s) : text;

		  if (! g_str_has_suffix(expanded, s))
		    g_string_append(answer, s);
		}
	    }
	  else
	    g_set_error(err,
			TRANSLATE_GENERIC_SERVICE_ERROR,
			TRANSLATE_GENERIC_SERVICE_ERROR_PARSE,
			_("unable to parse server data"));
	}

      g_free(response);
    }

  return answer ? g_string_free(answer, FALSE) : NULL;
}

char *
translate_generic_service_expand (const char *warning_prefix,
				  const char *str,
				  ...)
{
  GHashTable *subs;
  va_list args;
  const char *name;
  GString *result;
  int i;
  int dollar = -1;

  g_return_val_if_fail(warning_prefix != NULL, NULL);
  g_return_val_if_fail(str != NULL, NULL);

  va_start(args, str);
  subs = g_hash_table_new(g_str_hash, g_str_equal);

  while ((name = va_arg(args, const char *)))
    {
      const char *value;

      value = va_arg(args, const char *);
      g_return_val_if_fail(value != NULL, NULL);

      g_hash_table_insert(subs, (gpointer) name, (gpointer) value);
    }

  result = g_string_new(NULL);

  for (i = 0; str[i]; i++)
    if (dollar >= 0)
      {
	if (dollar == i - 1)
	  {
	    if (str[i] == '$')
	      {
		g_string_append_c(result, '$');
		dollar = -1;
	      }
	    else if (str[i] != '{')
	      {
		g_string_append_len(result, str + (i - 1), 2);
		dollar = -1;
	      }
	  }
	else
	  {
	    if (str[i] == '}')
	      {
		int start;
		char *variable;
		char *expanded;

		start = dollar + 2;

		variable = g_strndup(str + start, i - start);
		expanded = translate_generic_service_expand_variable(warning_prefix,
								     variable,
								     subs);
		g_free(variable);

		if (expanded)
		  {
		    g_string_append(result, expanded);
		    g_free(expanded);
		  }

		dollar = -1;
	      }
	  }
      }
    else
      {
	if (str[i] == '$')
	  dollar = i;
	else
	  g_string_append_c(result, str[i]);
      }

  g_hash_table_destroy(subs);
  va_end(args);

  return g_string_free(result, FALSE);
}

static char *
translate_generic_service_expand_variable (const char *warning_prefix,
					   const char *variable,
					   GHashTable *subs)
{
  char *s;
  char *varname;
  char **modifiers = NULL;
  char *expanded = NULL;

  g_return_val_if_fail(warning_prefix != NULL, NULL);
  g_return_val_if_fail(variable != NULL, NULL);
  g_return_val_if_fail(subs != NULL, NULL);

  s = strchr(variable, ':');
  if (s)
    {
      varname = g_strndup(variable, s - variable);
      modifiers = g_strsplit(s + 1, ",", 0);
    }
  else
    varname = g_strdup(variable);

  if (! strcmp(varname, "time"))
    {
      time_t now;

      now = translate_time();
      expanded = g_strdup_printf("%ul", now);
    }
  else
    expanded = g_strdup(g_hash_table_lookup(subs, varname));

  if (expanded)
    {
      if (modifiers)
	{
	  int i;

	  for (i = 0; modifiers[i]; i++)
	    {
	      char *mod_name;
	      char *mod_value;
	      char *modified;

	      s = strchr(modifiers[i], '=');
	      if (s)
		{
		  mod_name = g_strndup(modifiers[i], s - modifiers[i]);
		  mod_value = g_strdup(s + 1);
		}
	      else
		{
		  mod_name = g_strdup(modifiers[i]);
		  mod_value = NULL;
		}

	      modified = translate_generic_service_modify_value(warning_prefix,
								expanded,
								mod_name,
								mod_value);
	      g_free(mod_name);
	      g_free(mod_value);
	      
	      g_free(expanded);
	      expanded = modified;
	    }
	}
    }
  else
    g_warning(_("%s: unknown variable \"%s\""), warning_prefix, varname);

  g_free(varname);
  g_strfreev(modifiers);

  return expanded;
}

static char *
translate_generic_service_modify_value (const char *warning_prefix,
					const char *value,
					const char *modifier_name,
					const char *modifier_value)
{
  char *modified = NULL;

  g_return_val_if_fail(warning_prefix != NULL, NULL);
  g_return_val_if_fail(value != NULL, NULL);
  g_return_val_if_fail(modifier_name != NULL, NULL);

  if (! strcmp(modifier_name, "escape"))
    {
      if (modifier_value)
	g_warning(_("%s: value specified for \"escape\" modifier"), warning_prefix);
      modified = soup_uri_encode(value, NULL);
    }
  else if (! strcmp(modifier_name, "charset"))
    {
      if (modifier_value)
	{
	  GError *err = NULL;

	  modified = g_convert(value, -1, modifier_value, "UTF-8", NULL, NULL, &err);
	  if (! modified)
	    {
	      g_warning(_("%s: unable to convert to \"%s\": %s"), warning_prefix, modifier_value, err->message);
	      g_error_free(err);
	    }
	}
      else
	g_warning(_("%s: value of \"charset\" modifier missing"), warning_prefix);
    }
  else
    g_warning(_("%s: unknown modifier \"%s\""), warning_prefix, modifier_name);

  return modified ? modified : g_strdup(value);
}

static char *
translate_generic_service_translate_web_page (TranslateService *service,
					      const char *url,
					      const char *from,
					      const char *to,
					      TranslateProgressFunc progress_func,
					      gpointer user_data,
					      GError **err)
{
  TranslateGenericService *generic = TRANSLATE_GENERIC_SERVICE(service);
  TranslateGenericGroup *group;
  int group_pos;
  const char *service_from;
  const char *service_to;
  char *warning_prefix;
  char *translation_url;
  char *post = NULL;
  GSList *headers;
  char *response;

  group = translate_generic_service_get_group(generic, from, to, &group_pos);
  g_return_val_if_fail(group != NULL, NULL);

  service_from = translate_generic_group_get_service_tag(group, from);
  service_to = translate_generic_group_get_service_tag(group, to);

  warning_prefix = MAKE_WARNING_PREFIX(service, group_pos, "url", "web-page-translation");
  translation_url = translate_generic_service_expand(warning_prefix,
						     group->web_page_location->url,
						     "url", url,
						     "from", service_from,
						     "to", service_to,
						     NULL);
  g_free(warning_prefix);

  headers = g_slist_copy(group->http_headers);
  headers = g_slist_concat(headers, g_slist_copy(group->web_page_location->http_headers));

  if (group->web_page_location->post)
    {
      warning_prefix = MAKE_WARNING_PREFIX(service, group_pos, "post", "web-page-translation");
      post = translate_generic_service_expand(warning_prefix,
					      group->web_page_location->post,
					      "url", url,
					      "from", service_from,
					      "to", service_to,
					      NULL);
      g_free(warning_prefix);
    }
  else if (! headers)
    return translation_url;

  response = translate_generic_service_get(translation_url,
					   post,
					   group->web_page_location->content_type,
					   headers,
					   0,
					   progress_func,
					   user_data,
					   err);

  g_free(translation_url);
  translation_url = NULL;

  g_free(post);
  g_slist_free(headers);

  if (response)
    {
      int fd;
      char *filename;

      fd = g_file_open_tmp("libtranslate.XXXXXX", &filename, err);
      if (fd >= 0)
	{
	  GIOChannel *channel;

	  channel = g_io_channel_unix_new(fd);

	  if (g_io_channel_set_encoding(channel, NULL, err)
	      && g_io_channel_write_chars(channel, response, -1, NULL, err))
	    {
	      if (g_io_channel_shutdown(channel, TRUE, err))
		translation_url = g_strconcat("file://", filename, NULL);
	    }
	  else
	    g_io_channel_shutdown(channel, FALSE, NULL);

	  g_io_channel_unref(channel);
	  g_free(filename);
	}

      g_free(response);
    }
  
  return translation_url;
}

static SoupSession *
translate_generic_service_soup_session_sync_new (void)
{
  char *proxy_text_uri;
  SoupUri *proxy_uri = NULL;
  SoupSession *session;
  TranslateGenericSoupCookieJar *cookie_jar;

  proxy_text_uri = translate_get_proxy();
  if (proxy_text_uri)
    {
      proxy_uri = soup_uri_new(proxy_text_uri);
      if (! proxy_uri)
	g_warning(_("unable to parse proxy URI \"%s\""), proxy_text_uri);

      g_free(proxy_text_uri);
    }

  session = soup_session_sync_new_with_options(SOUP_SESSION_PROXY_URI, proxy_uri, NULL);

  if (proxy_uri)
    soup_uri_free(proxy_uri);

  cookie_jar = translate_generic_soup_cookie_jar_new();
  soup_session_add_filter(session, SOUP_MESSAGE_FILTER(cookie_jar));
  g_object_unref(cookie_jar);

  return session;
}

TranslateService *
translate_generic_service_new (const char *name,
			       const char *nick,
			       unsigned int max_chunk_len,
			       const GSList *groups)
{
  g_return_val_if_fail(name != NULL, NULL);
  g_return_val_if_fail(nick != NULL, NULL);

  return g_object_new(TRANSLATE_GENERIC_TYPE_SERVICE,
		      "name", name,
		      "nick", nick,
		      "max-chunk-len", max_chunk_len,
		      "groups", groups,
		      NULL);
}

GQuark
translate_generic_service_error_quark (void)
{
  return g_quark_from_static_string("translate-service-generic-error");
}
