//
//
//  BCGeneExpressionOmnibus.m
//  BioCocoa
//
//  Created by Scott Christley on 10/07/08.
//  Copyright (c) 2003-2009 The BioCocoa Project.
//  All rights reserved.
//
//  Redistribution and use in source and binary forms, with or without
//  modification, are permitted provided that the following conditions
//  are met:
//  1. Redistributions of source code must retain the above copyright
//  notice, this list of conditions and the following disclaimer.
//  2. Redistributions in binary form must reproduce the above copyright
//  notice, this list of conditions and the following disclaimer in the
//  documentation and/or other materials provided with the distribution.
//  3. The name of the author may not be used to endorse or promote products
//  derived from this software without specific prior written permission.
//
//  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
//  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
//  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
//  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
//  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
//  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
//  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
//  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
//  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
//  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#import "BCGeneExpressionOmnibus.h"
#import "BCPreferences.h"
#import "BCSeries.h"

#include <CoreServices/CoreServices.h>
#include <CoreFoundation/CoreFoundation.h>
#include <SystemConfiguration/SystemConfiguration.h>

#include <sys/dirent.h>
#include <sys/stat.h>
#include <unistd.h>         // getopt
#include <string.h>         // strmode
#include <stdlib.h>
#include <inttypes.h>


/* When using file streams, the 32KB buffer is probably not enough.
A good way to establish a buffer size is to increase it over time.
If every read consumes the entire buffer, start increasing the buffer
size, and at some point you would then cap it. 32KB is fine for network
sockets, although using the technique described above is still a good idea.
This sample avoids the technique because of the added complexity it
would introduce. */
#define kMyBufferSize  32768


/* MyStreamInfo holds the state of a particular operation (download, upload, or 
directory listing.  Some fields are only valid for some operations, as explained 
by their comments. */
typedef struct MyStreamInfo {

    CFWriteStreamRef  writeStream;              // download (destination file stream) and upload (FTP stream) only
    CFReadStreamRef   readStream;               // download (FTP stream), upload (source file stream), directory list (FTP stream)
    CFDictionaryRef   proxyDict;                // necessary to workaround <rdar://problem/3745574>, per discussion below
    NSMutableArray    *series;                  // hold the list of GEO series
    SInt64            fileSize;                 // download only, 0 indicates unknown
    UInt32            totalBytesWritten;        // download and upload only
    UInt32            leftOverByteCount;        // upload and directory list only, number of valid bytes at start of buffer
    UInt8             buffer[kMyBufferSize];    // buffer to hold left over bytes

} MyStreamInfo;


static const CFOptionFlags kNetworkEvents = 
      kCFStreamEventOpenCompleted
    | kCFStreamEventHasBytesAvailable
    | kCFStreamEventEndEncountered
    | kCFStreamEventCanAcceptBytes
    | kCFStreamEventErrorOccurred;
    

/* MyStreamInfoCreate creates a MyStreamInfo 'object' with the specified read and write stream. */
static void
MyStreamInfoCreate(MyStreamInfo ** info, CFReadStreamRef readStream, CFWriteStreamRef writeStream)
{
    MyStreamInfo * streamInfo;

    assert(info != NULL);
    assert(readStream != NULL);
    // writeStream may be NULL (this is the case for the directory list operation)
    
    streamInfo = malloc(sizeof(MyStreamInfo));
    assert(streamInfo != NULL);
    
    streamInfo->readStream        = readStream;
    streamInfo->writeStream       = writeStream;
    streamInfo->proxyDict         = NULL;           // see discussion of <rdar://problem/3745574> below
    streamInfo->fileSize          = 0;
    streamInfo->totalBytesWritten = 0;
    streamInfo->leftOverByteCount = 0;

    *info = streamInfo;
}


/* MyStreamInfoDestroy destroys a MyStreamInfo 'object', cleaning up any resources that it owns. */                                       
static void
MyStreamInfoDestroy(MyStreamInfo * info)
{
    assert(info != NULL);
    
    if (info->readStream) {
        CFReadStreamUnscheduleFromRunLoop(info->readStream, CFRunLoopGetCurrent(), kCFRunLoopCommonModes);
        (void) CFReadStreamSetClient(info->readStream, kCFStreamEventNone, NULL, NULL);
        
        /* CFReadStreamClose terminates the stream. */
        CFReadStreamClose(info->readStream);
        CFRelease(info->readStream);
    }

    if (info->writeStream) {
        CFWriteStreamUnscheduleFromRunLoop(info->writeStream, CFRunLoopGetCurrent(), kCFRunLoopCommonModes);
        (void) CFWriteStreamSetClient(info->writeStream, kCFStreamEventNone, NULL, NULL);
        
        /* CFWriteStreamClose terminates the stream. */
        CFWriteStreamClose(info->writeStream);
        CFRelease(info->writeStream);
    }

    if (info->proxyDict) {
        CFRelease(info->proxyDict);             // see discussion of <rdar://problem/3745574> below
    }
    
    free(info);
}


/* MyCFStreamSetUsernamePassword applies the specified user name and password to the stream. */
static void
MyCFStreamSetUsernamePassword(CFTypeRef stream, CFStringRef username, CFStringRef password)
{
    Boolean success;
    assert(stream != NULL);
    assert( (username != NULL) || (password == NULL) );
    
    if (username && CFStringGetLength(username) > 0) {

        if (CFGetTypeID(stream) == CFReadStreamGetTypeID()) {
            success = CFReadStreamSetProperty((CFReadStreamRef)stream, kCFStreamPropertyFTPUserName, username);
            assert(success);
            if (password) {
                success = CFReadStreamSetProperty((CFReadStreamRef)stream, kCFStreamPropertyFTPPassword, password);
                assert(success);
            }
        } else if (CFGetTypeID(stream) == CFWriteStreamGetTypeID()) {
            success = CFWriteStreamSetProperty((CFWriteStreamRef)stream, kCFStreamPropertyFTPUserName, username);
            assert(success);
            if (password) {
                success = CFWriteStreamSetProperty((CFWriteStreamRef)stream, kCFStreamPropertyFTPPassword, password);
                assert(success);
            }
        } else {
            assert(false);
        }
    }
}


/* MyCFStreamSetFTPProxy applies the current proxy settings to the specified stream.  This returns a 
reference to the proxy dictionary that we used because of <rdar://problem/3745574>, discussed below. */
static void
MyCFStreamSetFTPProxy(CFTypeRef stream, CFDictionaryRef * proxyDictPtr)
{
    CFDictionaryRef  proxyDict;
    CFNumberRef      passiveMode;
    CFBooleanRef     isPassive;
    Boolean          success;
    
    assert(stream != NULL);
    assert(proxyDictPtr != NULL);
    
    /* SCDynamicStoreCopyProxies gets the current Internet proxy settings.  Then we call
    CFReadStreamSetProperty, with property name kCFStreamPropertyFTPProxy, to apply the
    settings to the FTP read stream. */
    proxyDict = SCDynamicStoreCopyProxies(NULL);
    assert(proxyDict != NULL);    
    
    /* Get the FTP passive mode setting from the proxy dictionary.  Because of a bug <rdar://problem/3625438>
    setting the kCFStreamPropertyFTPProxy property using the SCDynamicStore proxy dictionary does not
    currently set the FTP passive mode setting on the stream, so we need to do it ourselves. 
    Also, <rdar://problem/4526438> indicates that out in the real world some people are setting 
    kSCPropNetProxiesFTPPassive to a Boolean, as opposed to a number.  That's just incorrect, 
    but I've hardened the code against it. */
    passiveMode = CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPPassive);
    if ( (passiveMode != NULL) && (CFGetTypeID(passiveMode) == CFNumberGetTypeID()) ) {
        int         value;
        
        success = CFNumberGetValue(passiveMode, kCFNumberIntType, &value);
        assert(success);
        
        if (value) isPassive = kCFBooleanTrue;
        else isPassive = kCFBooleanFalse;
    } else {
        assert(false);
        isPassive = kCFBooleanTrue;         // if prefs malformed, we just assume true
    }
        
    if (CFGetTypeID(stream) == CFReadStreamGetTypeID()) {
        success = CFReadStreamSetProperty((CFReadStreamRef)stream, kCFStreamPropertyFTPProxy, proxyDict);
        assert(success);
        success = CFReadStreamSetProperty((CFReadStreamRef)stream, kCFStreamPropertyFTPUsePassiveMode, isPassive);
        assert(success);
    } else if (CFGetTypeID(stream) == CFWriteStreamGetTypeID()) {
        success = CFWriteStreamSetProperty((CFWriteStreamRef)stream, kCFStreamPropertyFTPProxy, proxyDict);
        assert(success);
        success = CFWriteStreamSetProperty((CFWriteStreamRef)stream, kCFStreamPropertyFTPUsePassiveMode, isPassive);
        assert(success);
    } else {
        fprintf(stderr, "This is not a CFStream\n");
    }

    /* Prior to Mac OS X 10.4, CFFTPStream has a bug <rdar://problem/3745574> that causes it to reference the 
    proxy dictionary that you applied /after/ it has released its last reference to that dictionary.  This causes 
    a crash.  We work around this bug by holding on to our own reference to the proxy dictionary until we're 
    done with the stream.  Thus, our reference prevents the dictionary from being disposed, and thus CFFTPStream 
    can access it safely.  So, rather than release our reference to the proxy dictionary, we pass it back to 
    our caller and require it to release it. */
    
    // CFRelease(proxyDict);  After bug #3745574 is fixed, we'll be able to release the proxyDict here.
    
    *proxyDictPtr = proxyDict;
}


/* MyPrintDirectoryListing prints a FTP directory entry, represented by a CFDictionary 
as returned by CFFTPCreateParsedResourceListing, as a single line of text, much like 
you'd get from "ls -l". */
static void
MyPrintDirectoryListing(CFDictionaryRef dictionary, NSMutableArray *series)
{
    CFStringRef           cfName;

    assert(dictionary != NULL);

    /* Note that this sample assumes UTF-8 since that's what the Mac OS X
    FTP server returns, however, some servers may use a different encoding. */
    cfName = CFDictionaryGetValue(dictionary, kCFFTPResourceName);
    if (cfName) {
        assert(CFGetTypeID(cfName) == CFStringGetTypeID());
        char p0 = CFStringGetCharacterAtIndex(cfName, 0);
        char p1 = CFStringGetCharacterAtIndex(cfName, 1);
        char p2 = CFStringGetCharacterAtIndex(cfName, 2);
        if ((p0 == 'G') && (p1 == 'S') && (p2 == 'E')) {
          [series addObject: [NSString stringWithString: (NSString *)cfName]];
        }
        fprintf(stderr, "%s", [(NSString *)cfName UTF8String]);
    }

    fprintf(stderr, "\n");
}


/* MyDirectoryListingCallBack is the stream callback for the CFFTPStream during a directory 
list operation. Its main purpose is to read bytes off the FTP stream, which is returning bytes 
of the directory listing, parse them, and 'pretty' print the resulting directory entries. */
static void
MyDirectoryListingCallBack(CFReadStreamRef readStream, CFStreamEventType type, void * clientCallBackInfo)
{
    MyStreamInfo     *info = (MyStreamInfo *)clientCallBackInfo;
    CFIndex          bytesRead;
    CFStreamError    error;
    CFDictionaryRef  parsedDict;
    
    assert(readStream != NULL);
    assert(info       != NULL);
    assert(info->readStream == readStream);

    switch (type) {

        case kCFStreamEventOpenCompleted:
            fprintf(stderr, "Open complete\n");
            break;
        case kCFStreamEventHasBytesAvailable:
        
            /* When we get here, there are bytes to be read from the stream.  There are two cases:
            either info->leftOverByteCount is zero, in which case we complete processed the last 
            buffer full of data (or we're at the beginning of the listing), or 
            info->leftOverByteCount is non-zero, in which case there are that many bytes at the 
            start of info->buffer that were left over from the last time that we were called. 
            By definition, any left over bytes were insufficient to form a complete directory 
            entry.
            
            In both cases, we just read the next chunk of data from the directory listing stream 
            and append it to our buffer.  We then process the buffer to see if it now contains 
            any complete directory entries. */

            /* CFReadStreamRead will return the number of bytes read, or -1 if an error occurs
            preventing any bytes from being read, or 0 if the stream's end was encountered. */
            bytesRead = CFReadStreamRead(info->readStream, info->buffer + info->leftOverByteCount, kMyBufferSize - info->leftOverByteCount);
            if (bytesRead > 0) {
                const UInt8 *   nextByte;
                CFIndex         bytesRemaining;
                CFIndex         bytesConsumedThisTime;

                /* Parse directory entries from the buffer until we either run out of bytes 
                or we stop making forward progress (indicating that the buffer does not have 
                enough bytes of valid data to make a complete directory entry). */

                nextByte       = info->buffer;
                bytesRemaining = bytesRead + info->leftOverByteCount;
                do
                {                    

                    /* CFFTPCreateParsedResourceListing parses a line of file or folder listing
                    of Unix format, and stores the extracted result in a CFDictionary. */
                    bytesConsumedThisTime = CFFTPCreateParsedResourceListing(NULL, nextByte, bytesRemaining, &parsedDict);
                    if (bytesConsumedThisTime > 0) {

                        /* It is possible for CFFTPCreateParsedResourceListing to return a positive number 
                        but not create a parse dictionary.  For example, if the end of the listing text 
                        contains stuff that can't be parsed, CFFTPCreateParsedResourceListing returns 
                        a positive number (to tell the calle that it's consumed the data), but doesn't 
                        create a parse dictionary (because it couldn't make sens of the data).
                        So, it's important that we only try to print parseDict if it's not NULL. */
                        
                        if (parsedDict != NULL) {
                            MyPrintDirectoryListing(parsedDict, info->series);
                            CFRelease(parsedDict);
                        }

                        nextByte       += bytesConsumedThisTime;
                        bytesRemaining -= bytesConsumedThisTime;

                    } else if (bytesConsumedThisTime == 0) {
                        /* This should never happen because we supply a pretty large buffer. 
                        Still, we handle it by leaving the loop, which leaves the remaining 
                        bytes in the buffer. */
                    } else if (bytesConsumedThisTime == -1) {
                        fprintf(stderr, "CFFTPCreateParsedResourceListing parse failure\n");
                        goto exit;
                    }

                } while ( (bytesRemaining > 0) && (bytesConsumedThisTime > 0) );
                
                /* If any bytes were left over, leave them in the buffer for next time. */
                if (bytesRemaining > 0) {
                    memmove(info->buffer, nextByte, bytesRemaining);                    
                }
                info->leftOverByteCount = bytesRemaining;
            } else {
                /* If bytesRead < 0, we've hit an error.  If bytesRead == 0, we've hit the end of the 
                directory listing.  In either case, we do nothing, and rely on CF to call us with 
                kCFStreamEventErrorOccurred or kCFStreamEventEndEncountered in order for us to do our 
                clean up. */
            }
            break;
        case kCFStreamEventErrorOccurred:
            error = CFReadStreamGetError(info->readStream);
            NSLog(@"CFReadStreamGetError returned (%d, %ld)\n", error.domain, error.error);
            goto exit;
        case kCFStreamEventEndEncountered:
            fprintf(stderr, "Listing complete\n");
            goto exit;
        default:
            NSLog(@"Received unexpected CFStream event (%d)", type);
            break;
    }
    return;

exit:
    MyStreamInfoDestroy(info);
    CFRunLoopStop(CFRunLoopGetCurrent());
    return;
}


/* MySimpleDirectoryListing implements the directory list command.  It sets up a MyStreamInfo 
'object' with the read stream being an FTP stream of the directory to list and with no 
write stream.  It then returns, and the real work happens asynchronously in the runloop.  
The function returns true if the stream setup succeeded, and false if it failed. */
static Boolean
MySimpleDirectoryListing(CFStringRef urlString, CFStringRef username, CFStringRef password, NSMutableArray *anArray)
{
    CFReadStreamRef        readStream;
    CFStreamClientContext  context = { 0, NULL, NULL, NULL, NULL };
    CFURLRef               downloadURL;
    Boolean                success = true;
    MyStreamInfo           *streamInfo;

    assert(urlString != NULL);

    downloadURL = CFURLCreateWithString(kCFAllocatorDefault, urlString, NULL);
    assert(downloadURL != NULL);

    /* Create an FTP read stream for downloading operation from an FTP URL. */
    readStream = CFReadStreamCreateWithFTPURL(kCFAllocatorDefault, downloadURL);
    assert(readStream != NULL);
    CFRelease(downloadURL);
        
    /* Initialize our MyStreamInfo structure, which we use to store some information about the stream. */
    MyStreamInfoCreate(&streamInfo, readStream, NULL);
    streamInfo->series = anArray;
    context.info = (void *)streamInfo;

    /* CFReadStreamSetClient registers a callback to hear about interesting events that occur on a stream. */
    success = CFReadStreamSetClient(readStream, kNetworkEvents, MyDirectoryListingCallBack, &context);
    if (success) {

        /* Schedule a run loop on which the client can be notified about stream events.  The client
        callback will be triggered via the run loop.  It's the caller's responsibility to ensure that
        the run loop is running. */
        CFReadStreamScheduleWithRunLoop(readStream, CFRunLoopGetCurrent(), kCFRunLoopCommonModes);
        
        MyCFStreamSetUsernamePassword(readStream, username, password);
        MyCFStreamSetFTPProxy(readStream, &streamInfo->proxyDict);
    
        /* CFReadStreamOpen will return success/failure.  Opening a stream causes it to reserve all the
        system resources it requires.  If the stream can open non-blocking, this will always return TRUE;
        listen to the run loop source to find out when the open completes and whether it was successful. */
        success = CFReadStreamOpen(readStream);
        if (success == false) {
            fprintf(stderr, "CFReadStreamOpen failed\n");
            MyStreamInfoDestroy(streamInfo);
        }
    } else {
        fprintf(stderr, "CFReadStreamSetClient failed\n");
        MyStreamInfoDestroy(streamInfo);
    }

    return success;
}


@implementation BCGeneExpressionOmnibus

//
// Get list of all GEO series available from NCBI ftp
//
+ (NSMutableArray *)retrieveGEOSeriesList
{
  /* Retrieve the directory listing for the specified URL. */
  NSMutableArray *a = [NSMutableArray new];
  CFStringRef  username = NULL, password = NULL;
  CFStringRef dirPath = CFStringCreateWithCString(NULL, "ftp://ftp.ncbi.nih.gov/pub/geo/DATA/SOFT/by_series/", kCFStringEncodingUTF8);
  Boolean status = MySimpleDirectoryListing(dirPath, username, password, a);
  if (status) CFRunLoopRun();
  CFRelease(dirPath);
  if (!status) {
    [a release];
    return nil;
  } else {
    return [a autorelease];
  }
}

//
// Get data files from Gene Omnibus
//
- (void)startDownloadingURL: (NSString *)aName
{
	printf("downloading %s\n", [aName UTF8String]);
  // create the request
  NSURLRequest *theRequest=[NSURLRequest requestWithURL:[NSURL URLWithString:aName]
                                            cachePolicy:NSURLRequestUseProtocolCachePolicy
                                        timeoutInterval:60.0];
  // create the connection with the request
  // and start loading the data
	NSURLDownload  *theDownload=[[NSURLDownload alloc] initWithRequest:theRequest delegate:self];
  if (!theDownload) {
    // inform the user that the download could not be made
		NSLog(@"Download failed.");
  }
}

- (void)download:(NSURLDownload *)download decideDestinationWithSuggestedFilename:(NSString *)filename
{
  NSString *destinationFilename;

  // Save in shared gene expression directory
  destinationFilename = [[BCPreferences sharedDataSubdirectory: @"GeneExpression"]
        stringByAppendingPathComponent:filename];
  [download setDestination:destinationFilename allowOverwrite:YES];
}


- (void)download:(NSURLDownload *)download didFailWithError:(NSError *)error
{
  // release the connection
  [download release];
  
  // inform the user
  NSLog(@"Download failed! Error - %@ %@",
        [error localizedDescription],
        [[error userInfo] objectForKey:NSErrorFailingURLStringKey]);
}

- (void)downloadDidFinish:(NSURLDownload *)download
{
  // release the connection
  [download release];
  
  // Add the series if necessary
  BCSeries *aSeries = [BCSeries seriesWithId: seriesName];
  if (!aSeries) {
    aSeries = [[BCSeries alloc] initWithId: seriesName];
    [BCSeries addSeries: aSeries];
  }
}


- (void)downloadGEOSeries:(NSString *)anAccession
{
  // save the series name
  seriesName = anAccession;

	// ftp from Gene Omnibus
	NSMutableString *s = [NSMutableString string];
	[s appendString: @"ftp://ftp.ncbi.nih.gov/pub/geo/DATA/SOFT/by_series/"];
	[s appendString: anAccession];
	[s appendString: @"/"];
	[s appendString: anAccession];
	[s appendString: @"_family.soft.gz"];
	[self startDownloadingURL: s];
}

@end
