/**
 * @file   storage_manager.cc
 *
 * @section LICENSE
 *
 * The MIT License
 * 
 * @copyright Copyright (c) 2016 MIT and Intel Corporation
 * @copyright Copyright (c) 2018-2021 Omics Data Automation, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 * 
 * @section DESCRIPTION
 *
 * This file implements the StorageManager class.
 */

#include "storage_manager.h"

#include "uri.h"
#include "utils.h"
#include "storage_fs.h"

#include <cassert>
#include <cstring>
#include <dirent.h>
#include <fcntl.h>
#include <iostream>
#include <string>
#include <sys/stat.h>
#include <unistd.h>

/* ****************************** */
/*             MACROS             */
/* ****************************** */

#ifdef TILEDB_VERBOSE
#  define PRINT_ERROR(x) std::cerr << TILEDB_SM_ERRMSG << x << ".\n" 
#else
#  define PRINT_ERROR(x) do { } while(0) 
#endif

#if defined HAVE_OPENMP && defined USE_PARALLEL_SORT
  #include <parallel/algorithm>
  #define SORT_LIB __gnu_parallel
#else
  #include <algorithm>
  #define SORT_LIB std 
#endif

#define SORT_2(first, last) SORT_LIB::sort((first), (last))
#define SORT_3(first, last, comp) SORT_LIB::sort((first), (last), (comp))
#define GET_MACRO(_1, _2, _3, NAME, ...) NAME
#define SORT(...) GET_MACRO(__VA_ARGS__, SORT_3, SORT_2)(__VA_ARGS__)




/* ****************************** */
/*        GLOBAL VARIABLES        */
/* ****************************** */

std::string tiledb_sm_errmsg = "";




/* ****************************** */
/*   CONSTRUCTORS & DESTRUCTORS   */
/* ****************************** */

StorageManager::StorageManager() {
}

StorageManager::~StorageManager() {
  if (config_ != NULL)
     delete config_;
}




/* ****************************** */
/*             MUTATORS           */
/* ****************************** */

int StorageManager::finalize() {
  return open_array_mtx_destroy();
}

int StorageManager::init(StorageManagerConfig* config) {
  // Set configuration parameters
  if(config_set(config) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Initialize mutexes and return
  return open_array_mtx_init();
}

StorageManagerConfig* StorageManager::get_config() {
  return config_;
}

/* ****************************** */
/*            WORKSPACE           */
/* ****************************** */

int StorageManager::workspace_create(const std::string& workspace) {
  // Check if the workspace is inside a workspace or another group
  std::string parent_dir = ::parent_dir(fs_, workspace);
  if(is_workspace(fs_, parent_dir) || 
     is_group(fs_, parent_dir) ||
     is_array(fs_, parent_dir) ||
     is_metadata(fs_, parent_dir)) {
    std::string errmsg =
        "The workspace cannot be contained in another workspace, "
        "group, array or metadata directory";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Create workspace directory
  if(fs_->create_dir(workspace) != TILEDB_UT_OK) {  
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR;
  }

  // Create workspace file
  if(create_workspace_file(workspace) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Success
  return TILEDB_SM_OK;
}

static std::vector<std::string> list_workspaces(StorageFS *fs, const char *parent_dir) {
  std::vector<std::string> workspace_dirs;
  std::vector<std::string> all_dirs = ::get_dirs(fs, parent_dir);
  for(auto const& dir: all_dirs) {
    if(is_workspace(fs, dir)) {
      workspace_dirs.push_back(dir);
    } else if (fs->is_dir(dir) && !is_group(fs, dir) && !is_array(fs, dir) && !is_metadata(fs, dir)) {
      std::vector<std::string> list = list_workspaces(fs, dir.c_str());
      workspace_dirs.insert(std::end(workspace_dirs), std::begin(list), std::end(list));
    }
  }
  return workspace_dirs;
}

static std::string relative_dir(std::string dir, const char *parent_dir) {
  if (dir.find(parent_dir) != std::string::npos && dir.size() > strlen(parent_dir)) {
    return dir.substr(strlen(parent_dir)+1);
  } else if (strstr(parent_dir, "://")) {
    uri path_uri(parent_dir);
    std::string path;
    if (path_uri.path().size() > 1) {
      if (path_uri.path()[0] == '/') {
        path = path_uri.path().substr(1);
      } else {
        path = path_uri.path();
      }
      if (dir.find(path) != std::string::npos && dir.size() > path.size()) {
        if (path[path.size()-1] == '/') {
          return dir.substr(path.size());
        } else {
          return dir.substr(path.size()+1);
        }
      }
    }
  }
  return dir;
}

int StorageManager::ls_workspaces(
    const char *parent_dir,
    char** workspaces,
    int& workspace_num) {

  std::vector<std::string> workspace_dirs = list_workspaces(fs_, parent_dir);
  if (workspace_dirs.size() > static_cast<std::vector<std::string>::size_type>(workspace_num)) {
    std::string errmsg = "Cannot list TileDB workspaces; Directory buffer overflow";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  workspace_num = 0;
  for (auto dir: workspace_dirs) {
    strncpy(workspaces[workspace_num++], relative_dir(dir, parent_dir).c_str(), TILEDB_NAME_MAX_LEN);
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::ls_workspaces_c(const char* parent_dir, int& workspace_num) {
  // Get real parent directory
  std::string parent_dir_real = ::real_dir(fs_, parent_dir);

  // Initialize workspace number
  workspace_num = 0;

  std::vector<std::string> dirs = ::get_dirs(fs_, parent_dir);
  for(auto const& dir: dirs) {
    if (is_workspace(fs_, dir)) {
	workspace_num++;
    } else if (fs_->is_dir(dir) && !is_group(fs_, dir) && !is_array(fs_, dir) && !is_metadata(fs_, dir)) {
      int num = 0;
      if (ls_workspaces_c(dir.c_str(), num) == TILEDB_SM_OK) {
	workspace_num += num;
      }
    }
  }

  // Success
  return TILEDB_SM_OK;
}



/* ****************************** */
/*             GROUP              */
/* ****************************** */

int StorageManager::group_create(const std::string& group) const {
  // Check if the group is inside a workspace or another group
  std::string parent_dir = ::parent_dir(fs_, group);
  if(!is_workspace(fs_, parent_dir) && !is_group(fs_, parent_dir)) {
    std::string errmsg = 
        "The group must be contained in a workspace "
        "or another group";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Create group directory
  if(create_dir(fs_, group) != TILEDB_UT_OK) { 
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR;
  }

  // Create group file
  if(create_group_file(group) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Success
  return TILEDB_SM_OK;
}




/* ****************************** */
/*             ARRAY              */
/* ****************************** */

int StorageManager::array_consolidate(const char* array_dir, size_t buffer_size, int batch_size) {
  // Create an array object
  Array* array;
  if(array_init(
      array,
      array_dir,
      TILEDB_ARRAY_CONSOLIDATE,
      NULL,
      NULL,
      0) != TILEDB_SM_OK) 
    return TILEDB_SM_ERR;

  // Consolidate array
  Fragment* new_fragment;
  std::vector<std::string> old_fragment_names;
  int rc_array_consolidate = 
      array->consolidate(new_fragment, old_fragment_names, buffer_size, batch_size);
  
  // Close the array
  int rc_array_close = array_close(array->get_array_path_used());

  // Finalize consolidation
  int rc_consolidation_finalize = 
      consolidation_finalize(new_fragment, old_fragment_names);

  // Finalize array
  int rc_array_finalize = array->finalize();
  delete array;

  int rc_delete = delete_directories(fs_, old_fragment_names);

  // Errors 
  if(rc_array_consolidate != TILEDB_AR_OK) {
    tiledb_sm_errmsg = tiledb_ar_errmsg;
    return TILEDB_SM_ERR;
  }
  if(rc_array_close != TILEDB_SM_OK              ||
     rc_array_finalize != TILEDB_SM_OK           ||
     rc_consolidation_finalize != TILEDB_SM_OK   ||
     rc_delete != TILEDB_UT_OK)
    return TILEDB_SM_ERR;

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::array_create(const ArraySchemaC* array_schema_c) const {
  // Initialize array schema
  ArraySchema* array_schema = new ArraySchema(fs_);
  if(array_schema->init(array_schema_c) != TILEDB_AS_OK) {
    delete array_schema;
    tiledb_sm_errmsg = tiledb_as_errmsg;
    return TILEDB_SM_ERR;
  }

  // Get real array directory name
  std::string dir = array_schema->array_name();
  
  std::string parent_dir = ::parent_dir(fs_, dir);

  // Check if the array directory is contained in a workspace, group or array
  if(!is_workspace(fs_, parent_dir) && 
     !is_group(fs_, parent_dir)) {
    std::string errmsg = 
        std::string("Cannot create array; Directory '") + parent_dir + 
        "' must be a TileDB workspace or group";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Create array with the new schema
  int rc = array_create(array_schema);
  
  // Clean up
  delete array_schema;

  // Return
  if(rc == TILEDB_SM_OK)
    return TILEDB_SM_OK;
  else
    return TILEDB_SM_ERR;
}

int StorageManager::array_create(const ArraySchema* array_schema) const {
  // Check array schema
  if(array_schema == NULL) {
    std::string errmsg = "Cannot create array; Empty array schema";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Create array directory
  std::string dir = array_schema->array_name();
  if(create_dir(fs_, dir) != TILEDB_UT_OK) { 
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR;
  }

  // Store array schema
  if(array_store_schema(dir, array_schema) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Create consolidation filelock
  if(consolidation_filelock_create(dir) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Success
  return TILEDB_SM_OK;
}

void StorageManager::array_get_fragment_names(
    const std::string& array,
    std::vector<std::string>& fragment_names) {

  // Get directory names in the array folder
  fragment_names = get_fragment_dirs(fs_, real_dir(fs_, array)); 

  // Sort the fragment names
  sort_fragment_names(fragment_names);
}

int StorageManager::array_load_book_keeping(
    const ArraySchema* array_schema,
    const std::vector<std::string>& fragment_names,
    std::vector<BookKeeping*>& book_keeping,
    int mode) {
  // For easy reference
  int fragment_num = fragment_names.size(); 

  // Initialization
  book_keeping.resize(fragment_num);

  // Load the book-keeping for each fragment
  for(int i=0; i<fragment_num; ++i) {
    // For easy reference
    int dense = 
        !fs_->is_file(fs_->append_paths(fragment_names[i], std::string(TILEDB_COORDS) + TILEDB_FILE_SUFFIX));

    // Create new book-keeping structure for the fragment
    BookKeeping* f_book_keeping = 
        new BookKeeping(
            array_schema, 
            dense, 
            fragment_names[i], 
            mode);

    // Load book-keeping
    if(f_book_keeping->load(fs_) != TILEDB_BK_OK) {
      delete f_book_keeping;
      tiledb_sm_errmsg = tiledb_bk_errmsg;
      return TILEDB_SM_ERR;
    }

    // Append to the open array entry
    book_keeping[i] = f_book_keeping;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::array_load_schema(
    const char* array_dir,
    ArraySchema*& array_schema) const {
  // Get real array path
  std::string real_array_dir = ::real_dir(fs_, array_dir);

  // Check if array exists
  if(!is_array(fs_, real_array_dir)) {
    std::string errmsg = 
        std::string("Cannot load array schema; Array '") + 
        real_array_dir + "' does not exist";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  std::string filename = fs_->append_paths(real_array_dir, TILEDB_ARRAY_SCHEMA_FILENAME);

  // Initialize buffer
  ssize_t buffer_size =  file_size(fs_, filename);
  assert(buffer_size > 0);

  void* buffer = malloc(buffer_size);
  if (buffer == NULL) {
    std::string errmsg = "Storage Manager memory allocation error";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Load array schema
  if(read_from_file(fs_, filename, 0, buffer, buffer_size) == TILEDB_UT_ERR) {
    std::string errmsg = "Cannot load array schema; File reading error";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    free(buffer);
    return TILEDB_SM_ERR;
  } 

  // Initialize array schema
  array_schema = new ArraySchema(fs_);
  if(array_schema->deserialize(buffer, buffer_size) != TILEDB_AS_OK) {
    free(buffer);
    delete array_schema;
    tiledb_sm_errmsg = tiledb_as_errmsg;
    return TILEDB_SM_ERR;
  }

  //Old TileDB version - create consolidate file
  if(!(array_schema->version_tag_exists())) {
    std::string filename = fs_->append_paths(real_array_dir, TILEDB_SM_CONSOLIDATION_FILELOCK_NAME);
    if (create_file(fs_, filename,  O_WRONLY | O_CREAT | O_SYNC, S_IRWXU) == TILEDB_UT_ERR) {
      std::string errmsg = "Cannot create consolidation file for old tiledb support";
      tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
      return TILEDB_SM_ERR;
    }
  }

  // Clean up
  close_file(fs_, filename);
  free(buffer);

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::array_init(
    Array*& array,
    const char* array_dir,
    int mode,
    const void* subarray,
    const char** attributes,
    int attribute_num)  {
  // Check array name length
  if(array_dir == NULL || strlen(array_dir) > TILEDB_NAME_MAX_LEN) {
    std::string errmsg = "Invalid array name length";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Load array schema
  ArraySchema* array_schema;
  if(array_load_schema(array_dir, array_schema) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  std::string full_array_path = real_dir(fs_, array_dir);

  // Open the array
  OpenArray* open_array = NULL;
  if(array_read_mode(mode) || array_consolidate_mode(mode)) {
    if(array_open(full_array_path, open_array, mode) != TILEDB_SM_OK)
      return TILEDB_SM_ERR;
  }

  Array* array_clone = NULL;
  if (!array_consolidate_mode(mode)) {
    // Create the clone Array object. No need to clone for consolidation
    array_clone = new Array();
    int rc_clone = array_clone->init(
        array_schema,
        full_array_path,
        open_array->fragment_names_,
        open_array->book_keeping_,
        mode,
        attributes,
        attribute_num,
        subarray,
        config_);

    // Handle error
    if(rc_clone != TILEDB_AR_OK) {
      delete array_schema;
      delete array_clone;
      array = NULL;
      if(array_read_mode(mode))
        array_close(full_array_path);
      tiledb_sm_errmsg = tiledb_ar_errmsg;
      return TILEDB_SM_ERR;
    }
  }

  // Create actual array
  array = new Array();
  int rc = array->init(
               array_schema,
               full_array_path,
               open_array->fragment_names_,
               open_array->book_keeping_,
               mode,
               attributes, 
               attribute_num, 
               subarray,
               config_,
               array_clone);

  // Handle error
  if(rc != TILEDB_AR_OK) {
    delete array_schema;
    delete array;
    array = NULL;
    if(array_read_mode(mode)) 
      array_close(full_array_path);
    tiledb_sm_errmsg = tiledb_ar_errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::array_finalize(Array* array) {
  // If the array is NULL, do nothing
  if(array == NULL)
    return TILEDB_SM_OK;

  // Finalize and close the array
  int rc_finalize = array->finalize();
  int rc_close = TILEDB_SM_OK;
  if(array->read_mode())
    rc_close = array_close(array->get_array_path_used());

  // Clean up
  delete array;

  // Errors
  if(rc_close != TILEDB_SM_OK)
    return TILEDB_SM_ERR;
  if(rc_finalize != TILEDB_AR_OK) {
    tiledb_sm_errmsg = tiledb_ar_errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::array_sync(Array* array) {
  // If the array is NULL, do nothing
  if(array == NULL)
    return TILEDB_SM_OK;

  // Sync array
  if(array->sync() != TILEDB_AR_OK) {
    tiledb_sm_errmsg = tiledb_ar_errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::array_sync_attribute(
    Array* array,
    const std::string& attribute) {
  // If the array is NULL, do nothing
  if(array == NULL)
    return TILEDB_SM_OK;

  // Sync array
  if(array->sync_attribute(attribute) != TILEDB_AR_OK) {
    tiledb_sm_errmsg = tiledb_ar_errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::array_iterator_init(
    ArrayIterator*& array_it,
    const char* array_dir,
    int mode,
    const void* subarray,
    const char** attributes,
    int attribute_num,
    void** buffers,
    size_t* buffer_sizes,
    const char* filter_expression) {
  // Create Array object. This also creates/updates an open array entry
  Array* array;
  if(array_init(
      array, 
      array_dir, 
      mode,
      subarray, 
      attributes, 
      attribute_num) != TILEDB_SM_OK) {
    array_it = NULL;
    return TILEDB_SM_ERR;
  }

  // Create ArrayIterator object
  array_it = new ArrayIterator();
  if(array_it->init(array, buffers, buffer_sizes, filter_expression) != TILEDB_AIT_OK) {
    array_finalize(array);
    delete array_it;
    array_it = NULL;
    tiledb_sm_errmsg = tiledb_ait_errmsg;
    return TILEDB_SM_ERR;
  } 

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::array_iterator_finalize(
    ArrayIterator* array_it) {
  // If the array iterator is NULL, do nothing
  if(array_it == NULL)
    return TILEDB_SM_OK;

  // Finalize and close array
  std::string array_name = array_it->array_name();
  int rc_finalize = array_it->finalize();
  int rc_close = array_close(array_name);

  // Clean up
  delete array_it;

  // Errors
  if(rc_finalize != TILEDB_AIT_OK) {
    tiledb_sm_errmsg = tiledb_ait_errmsg;
    return TILEDB_SM_ERR;
  }
  if(rc_close != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Success
  return TILEDB_SM_OK;
}




/* ****************************** */
/*            METADATA            */
/* ****************************** */

int StorageManager::metadata_consolidate(const char* metadata_dir) {
  // Load metadata schema
  ArraySchema* array_schema;
  if(metadata_load_schema(metadata_dir, array_schema) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Set attributes
  char** attributes;
  int attribute_num = array_schema->attribute_num();
    attributes = new char*[attribute_num+1];
    for(int i=0; i<attribute_num+1; ++i) {
      const char* attribute = array_schema->attribute(i).c_str();
      size_t attribute_len = strlen(attribute);
      attributes[i] = new char[attribute_len+1];
      strcpy(attributes[i], attribute);
    }

  // Create a metadata object
  Metadata* metadata;
  int rc_init = metadata_init(
                    metadata,
                    metadata_dir,
                    TILEDB_METADATA_READ,
                    (const char**) attributes,
                    attribute_num+1);

  // Clean up
  for(int i=0; i<attribute_num+1; ++i) 
    delete [] attributes[i];
  delete [] attributes;
  delete array_schema;

  if(rc_init != TILEDB_MT_OK) {
    tiledb_sm_errmsg = tiledb_mt_errmsg;
    return TILEDB_SM_ERR;
  }

  // Consolidate metadata
  Fragment* new_fragment;
  std::vector<std::string> old_fragment_names;
  int rc_metadata_consolidate = 
      metadata->consolidate(new_fragment, old_fragment_names);
  
  // Close the underlying array
  std::string array_name = metadata->array_schema()->array_name();
  int rc_array_close = array_close(array_name);

  // Finalize consolidation
  int rc_consolidation_finalize = 
      consolidation_finalize(new_fragment, old_fragment_names);

  // Finalize metadata
  int rc_metadata_finalize = metadata->finalize();
  delete metadata;

  int rc_delete = delete_directories(fs_, old_fragment_names);

  // Errors 
  if(rc_metadata_consolidate != TILEDB_MT_OK) {
    tiledb_sm_errmsg = tiledb_mt_errmsg;
    return TILEDB_SM_ERR;
  }
  if(rc_array_close != TILEDB_SM_OK            ||
     rc_metadata_finalize != TILEDB_SM_OK      ||
     rc_consolidation_finalize != TILEDB_SM_OK ||
     rc_delete != TILEDB_UT_OK
     )
    return TILEDB_SM_ERR;

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::metadata_create(
    const MetadataSchemaC* metadata_schema_c) const {
  // Initialize array schema
  ArraySchema* array_schema = new ArraySchema(fs_);
  if(array_schema->init(metadata_schema_c) != TILEDB_AS_OK) {
    delete array_schema;
    tiledb_sm_errmsg = tiledb_as_errmsg;
    return TILEDB_SM_ERR;
  }

  // Get real array directory name
  std::string dir = array_schema->array_name();
  std::string parent_dir = ::parent_dir(fs_, dir);

  // Check if the array directory is contained in a workspace, group or array
  if(!is_workspace(fs_, parent_dir) && 
     !is_group(fs_, parent_dir) &&
     !is_array(fs_, parent_dir)) {
    std::string errmsg = 
        std::string("Cannot create metadata; Directory '") + 
        parent_dir + "' must be a TileDB workspace, group, or array";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Create array with the new schema
  int rc = metadata_create(array_schema);
  
  // Clean up
  delete array_schema;

  // Return
  if(rc == TILEDB_SM_OK)
    return TILEDB_SM_OK;
  else
    return TILEDB_SM_ERR;
}

int StorageManager::metadata_create(const ArraySchema* array_schema) const {
  // Check metadata schema
  if(array_schema == NULL) {
    std::string errmsg = "Cannot create metadata; Empty metadata schema";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Create array directory
  std::string dir = array_schema->array_name();
  if(create_dir(fs_, dir) != TILEDB_UT_OK) { 
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR;
  }

  std::string filename = fs_->append_paths(dir, TILEDB_METADATA_SCHEMA_FILENAME);

  // Serialize metadata schema
  void* array_schema_bin;
  size_t array_schema_bin_size;
  if(array_schema->serialize(array_schema_bin, array_schema_bin_size) !=
     TILEDB_AS_OK) {
    tiledb_sm_errmsg = tiledb_as_errmsg;
    return TILEDB_SM_ERR;
  }

  // Store the array schema
  int rc = write_to_file(fs_, filename, array_schema_bin, array_schema_bin_size);
  if (!rc) {
    rc = close_file(fs_, filename);
  }
  if (rc) {
    free(array_schema_bin);
    std::string errmsg = std::string("Cannot create metadata");
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Clean up
  free(array_schema_bin);

  // Create consolidation filelock
  if(consolidation_filelock_create(dir) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::metadata_load_schema(
    const char* metadata_dir,
    ArraySchema*& array_schema) const {
  // Get real array path
  std::string real_metadata_dir = ::real_dir(fs_, metadata_dir);

  // Check if metadata exists
  if(!is_metadata(fs_, real_metadata_dir)) {
    PRINT_ERROR(std::string("Cannot load metadata schema; Metadata '") + 
                real_metadata_dir + "' does not exist");
    return TILEDB_SM_ERR;
  }

  // Open array schema file 
  std::string filename = fs_->append_paths(real_metadata_dir, TILEDB_METADATA_SCHEMA_FILENAME);

  // Initialize buffer
  ssize_t buffer_size = file_size(fs_, filename);
  if(buffer_size == 0) {
    std::string errmsg = 
        "Cannot load metadata schema; Empty metadata schema file";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }
  void* buffer = malloc(buffer_size);

  // Load array schema
  if (read_from_file(fs_, filename, 0, buffer, buffer_size) == TILEDB_UT_ERR) {
    free(buffer);
    std::string errmsg = "Cannot load metadata schema; File reading error";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_UT_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  } 

  // Initialize array schema
  array_schema = new ArraySchema(fs_);
  if(array_schema->deserialize(buffer, buffer_size) == TILEDB_AS_ERR) {
    free(buffer);
    delete array_schema;
    tiledb_sm_errmsg = tiledb_as_errmsg;
    return TILEDB_SM_ERR;
  }

  // Clean up
  close_file(fs_, filename);
  free(buffer);

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::metadata_init(
    Metadata*& metadata,
    const char* metadata_dir,
    int mode,
    const char** attributes,
    int attribute_num)  {
  // Check metadata name length
  if(metadata_dir == NULL || strlen(metadata_dir) > TILEDB_NAME_MAX_LEN) {
    std::string errmsg = "Invalid metadata name length";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Load metadata schema
  ArraySchema* array_schema;
  if(metadata_load_schema(metadata_dir, array_schema) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Open the array that implements the metadata
  OpenArray* open_array = NULL;
  if(mode == TILEDB_METADATA_READ) {
    if(array_open(
           real_dir(fs_, metadata_dir), 
           open_array, 
           TILEDB_ARRAY_READ) != TILEDB_SM_OK)
      return TILEDB_SM_ERR;
  }

  // Create metadata object
  metadata = new Metadata();
  int rc = metadata->init(
               array_schema, 
               open_array->fragment_names_,
               open_array->book_keeping_,
               mode, 
               attributes, 
               attribute_num,
               config_);

  // Return
  if(rc != TILEDB_MT_OK) {
    delete array_schema;
    delete metadata;
    metadata = NULL;
    array_close(metadata_dir);
    tiledb_sm_errmsg = tiledb_mt_errmsg;
    return TILEDB_SM_ERR;
  } else {
    return TILEDB_SM_OK;
  }
}

int StorageManager::metadata_finalize(Metadata* metadata) {
  // If the metadata is NULL, do nothing
  if(metadata == NULL)
    return TILEDB_SM_OK;

  // Finalize the metadata and close the underlying array
  std::string array_name = metadata->array_schema()->array_name();
  int mode = metadata->array()->mode();
  int rc_finalize = metadata->finalize();
  int rc_close = TILEDB_SM_OK;
  if(mode == TILEDB_METADATA_READ)
    rc_close = array_close(array_name);

  // Clean up
  delete metadata;

  // Errors
  if(rc_close != TILEDB_SM_OK)
    return TILEDB_SM_ERR;
  if(rc_finalize != TILEDB_MT_OK) {
    tiledb_sm_errmsg = tiledb_mt_errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::metadata_iterator_init(
    MetadataIterator*& metadata_it,
    const char* metadata_dir,
    const char** attributes,
    int attribute_num,
    void** buffers,
    size_t* buffer_sizes) {
  // Create metadata object
  Metadata* metadata;
  if(metadata_init(
         metadata, 
         metadata_dir,
         TILEDB_METADATA_READ, 
         attributes, 
         attribute_num) != TILEDB_SM_OK) {
    metadata_it = NULL;
    return TILEDB_SM_ERR;
  } 

  // Create MetadataIterator object
  metadata_it = new MetadataIterator();
  if(metadata_it->init(metadata, buffers, buffer_sizes) != TILEDB_MIT_OK) {
    metadata_finalize(metadata);
    delete metadata_it;
    metadata_it = NULL;
    tiledb_sm_errmsg = tiledb_mit_errmsg;
    return TILEDB_SM_ERR;
  } 

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::metadata_iterator_finalize(
    MetadataIterator* metadata_it) {
  // If the metadata iterator is NULL, do nothing
  if(metadata_it == NULL)
    return TILEDB_SM_OK;

  // Close array and finalize metadata
  std::string metadata_name = metadata_it->metadata_name();
  int rc_finalize = metadata_it->finalize();
  int rc_close = array_close(metadata_name);

  // Clean up
  delete metadata_it;

  // Errors
  if(rc_finalize != TILEDB_MIT_OK) {
    tiledb_sm_errmsg = tiledb_mit_errmsg;
    return TILEDB_SM_ERR;
  }
  if(rc_close != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Success
  return TILEDB_SM_OK;
}




/* ****************************** */
/*               MISC             */
/* ****************************** */

int StorageManager::ls(
    const char* parent_dir,
    char** dirs, 
    int* dir_types,
    int& dir_num) const {
  // Initialize directory counter
  int dir_i = 0;
  int dir_type;

  std::vector<std::string> all_dirs = ::get_dirs(fs_, parent_dir);
  for(auto const& dir: all_dirs) {
    if(is_workspace(fs_, dir)) {
      dir_type = TILEDB_WORKSPACE;
    } else if(is_group(fs_, dir)) {
      dir_type = TILEDB_GROUP;
    } else if(is_metadata(fs_, dir)) {
      dir_type = TILEDB_METADATA;
    } else if(is_array(fs_, dir)){
      dir_type = TILEDB_ARRAY;
    } else {
      dir_type = -1;
    }
    if (dir_type >= 0) {
      if (dir_i < dir_num) {
	strncpy(dirs[dir_i], relative_dir(dir, parent_dir).c_str(), TILEDB_NAME_MAX_LEN);
	dir_types[dir_i++] = dir_type;
      } else {
	std::string errmsg = 
            "Cannot list entire TileDB directory; Directory buffer overflow";
        PRINT_ERROR(errmsg);
        tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
        return TILEDB_SM_ERR;
      }
    }
  }

  // Set the number of directories
  dir_num = dir_i;

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::ls_c(const char* parent_dir, int& dir_num) const {
  // Get real parent directory
  std::string parent_dir_real = ::real_dir(fs_, parent_dir); 

  // Initialize directory number
  dir_num =0;

  std::vector<std::string> dirs = ::get_dirs(fs_, parent_dir);
  for(auto const& dir: dirs) {
    if(is_workspace(fs_, dir) || is_group(fs_, dir) || is_metadata(fs_, dir) || is_array(fs_, dir)){
      dir_num++;
    }
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::clear(const std::string& dir) const {
  if(is_workspace(fs_, dir)) {
    return workspace_clear(dir);
  } else if(is_group(fs_, dir)) {
    return group_clear(dir);
  } else if(is_array(fs_, dir)) {
    return array_clear(dir);
  } else if(is_metadata(fs_, dir)) {
    return metadata_clear(dir);
  } else {
    std::string errmsg = "Clear failed; Invalid directory";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }
}

int StorageManager::delete_entire(const std::string& dir) {
  if(is_workspace(fs_, dir)) {
    return workspace_delete(dir);
  } else if(is_group(fs_, dir)) {
    return group_delete(dir);
  } else if(is_array(fs_, dir)) {
    return array_delete(dir);
  } else if(is_metadata(fs_, dir)) {
    return metadata_delete(dir);
  } else {
    std::string errmsg = "Delete failed; Invalid directory";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::move(
    const std::string& old_dir,
    const std::string& new_dir) {
  if(is_workspace(fs_, old_dir)) {
    return workspace_move(old_dir, new_dir);
  } else if(is_group(fs_, old_dir)) {
    return group_move(old_dir, new_dir);
  } else if(is_array(fs_, old_dir)) {
    return array_move(old_dir, new_dir);
  } else if(is_metadata(fs_, old_dir)) {
    return metadata_move(old_dir, new_dir);
  } else {
    std::string errmsg = "Move failed; Invalid source directory";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}




/* ****************************** */
/*         PRIVATE METHODS        */
/* ****************************** */

int StorageManager::array_clear(
    const std::string& array) const {
  // Get real array directory name
  std::string array_real = ::real_dir(fs_, array);

  // Check if the array exists
  if(!is_array(fs_, array_real)) {
    std::string errmsg = 
        std::string("Array '") + array_real + 
        "' does not exist";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  std::vector<std::string> all_dirs = ::get_dirs(fs_, array_real);
  for(auto const& dir: all_dirs) {
    if(is_metadata(fs_, dir)) {
      metadata_delete(dir);
    } else if(is_fragment(fs_, dir)) {
      delete_dir(fs_, dir);
    } else {
      std::string errmsg =
          std::string("Cannot delete non TileDB related element '") +
          dir + "'";
      PRINT_ERROR(errmsg);
      tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
      return TILEDB_SM_ERR;
    }
  }
  
  // Success
  return TILEDB_SM_OK;
}

int StorageManager::array_close(const std::string& array) {
  // Lock mutexes
  if(open_array_mtx_lock() != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Find the open array entry
  std::map<std::string, OpenArray*>::iterator it = 
      open_arrays_.find(real_dir(fs_, array));

  // Sanity check
  if(it == open_arrays_.end()) { 
    std::string errmsg = "Cannot close array; Open array entry not found";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Lock the mutex of the array
  if(it->second->mutex_lock() != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Decrement counter
  --(it->second->cnt_);

  // Delete open array entry if necessary
  int rc_mtx_destroy = TILEDB_SM_OK;
  int rc_filelock = TILEDB_SM_OK;
  if(it->second->cnt_ == 0) {
    // Clean up book-keeping
    std::vector<BookKeeping*>::iterator bit = it->second->book_keeping_.begin();
    for(; bit != it->second->book_keeping_.end(); ++bit) 
      delete *bit;

    // Unlock and destroy mutexes
    it->second->mutex_unlock();
    rc_mtx_destroy = it->second->mutex_destroy();

    // Unlock consolidation filelock
    rc_filelock = consolidation_filelock_unlock(
                      it->second->consolidation_filelock_);

    // Delete array schema
    if(it->second->array_schema_ != NULL)
      delete it->second->array_schema_;

    // Free open array
    delete it->second;

    // Delete open array entry
    open_arrays_.erase(it);
  } else { 
    // Unlock the mutex of the array
    if(it->second->mutex_unlock() != TILEDB_SM_OK)
      return TILEDB_SM_ERR;
  }

  // Unlock mutexes
  int rc_mtx_unlock = open_array_mtx_unlock();

  // Return
  if(rc_mtx_destroy != TILEDB_SM_OK || 
     rc_filelock != TILEDB_SM_OK    ||
     rc_mtx_unlock != TILEDB_SM_OK)
    return TILEDB_SM_ERR;
  else
    return TILEDB_SM_OK;
}

int StorageManager::array_delete(
    const std::string& array) const {
  // Clear the array
  if(array_clear(array) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Delete array directory
  if(delete_dir(fs_, array) != TILEDB_UT_OK) {
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR; 
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::array_get_open_array_entry(
    const std::string& array,
    OpenArray*& open_array,
    bool& opened_first_time) {
  // Lock mutexes
  if(open_array_mtx_lock() != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Find the open array entry
  std::map<std::string, OpenArray*>::iterator it = open_arrays_.find(array);
  // Create and init entry if it does not exist
  if(it == open_arrays_.end()) { 
    open_array = new OpenArray();
    open_array->cnt_ = 0;
    open_array->consolidation_filelock_ = -1;
    open_array->book_keeping_ = std::vector<BookKeeping*>();
    if(open_array->mutex_init() != TILEDB_SM_OK) {
      open_array->mutex_unlock();
      return TILEDB_SM_ERR;
    }
    open_arrays_[array] = open_array; 
    opened_first_time = true;
  } else {
    open_array = it->second;
    opened_first_time = false;
  }

  // Increment counter
  ++(open_array->cnt_);

  // Unlock mutexes
  if(open_array_mtx_unlock() != TILEDB_SM_OK) {
    --open_array->cnt_;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::array_move(
    const std::string& old_array,
    const std::string& new_array) const {
  // Get real array directory name
  std::string old_array_real = real_dir(fs_, old_array);
  std::string new_array_real = real_dir(fs_, new_array);

  // Check if the old array exists
  if(!is_array(fs_, old_array_real)) {
    std::string errmsg = 
        std::string("Array '") + old_array_real + 
        "' does not exist";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Make sure that the new array is not an existing directory
  if(is_dir(fs_, new_array_real)) {
    std::string errmsg = 
        std::string("Directory '") + 
        new_array_real + "' already exists";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Check if the new array are inside a workspace or group
  std::string new_array_parent_folder = parent_dir(fs_, new_array_real);
  if(!is_group(fs_, new_array_parent_folder) && 
     !is_workspace(fs_, new_array_parent_folder)) {
    std::string errmsg = 
        std::string("Folder '") + new_array_parent_folder + "' must "
        "be either a workspace or a group";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Rename array
  if(move_path(fs_, old_array_real, new_array_real)) {
    std::string errmsg = 
        std::string("Cannot move array; ") + strerror(errno);
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Incorporate new name in the array schema
  ArraySchema* array_schema;
  if(array_load_schema(new_array_real.c_str(), array_schema) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;
  array_schema->set_array_name(new_array_real.c_str());

  // Store the new schema
  if(array_store_schema(new_array_real, array_schema) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Clean up
  delete array_schema;

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::array_open(
    const std::string& array_name, 
    OpenArray*& open_array,
    int mode) {
  auto opened_first_time = false;

  // Get the open array entry
  if(array_get_open_array_entry(array_name, open_array, opened_first_time) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Lock the mutex of the array
  if(open_array->mutex_lock() != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // First time the array is opened
  if(opened_first_time) {
    // Acquire shared lock on consolidation filelock
    if(consolidation_filelock_lock(
        array_name,
        open_array->consolidation_filelock_, 
        TILEDB_SM_SHARED_LOCK) != TILEDB_SM_OK) {
      open_array->mutex_unlock();
      return TILEDB_SM_ERR;
    }

    // Get the fragment names
    array_get_fragment_names(array_name, open_array->fragment_names_);

    // Get array schema
    if(is_array(fs_, array_name)) { // Array
      if(array_load_schema(
             array_name.c_str(), 
             open_array->array_schema_) != TILEDB_SM_OK)
        return TILEDB_SM_ERR;
    } else {                  // Metadata
      if(metadata_load_schema(
             array_name.c_str(), 
             open_array->array_schema_) != TILEDB_SM_OK)
        return TILEDB_SM_ERR;
    }
  }

  if (!array_consolidate_mode(mode)) {
    // Load the book-keeping for each fragment. For consolidate mode, each fragment will be opened
    // separately
    if(array_load_book_keeping(
           open_array->array_schema_,
           open_array->fragment_names_,
           open_array->book_keeping_,
           mode) != TILEDB_SM_OK) {
      delete open_array->array_schema_;
      open_array->array_schema_ = NULL;
      open_array->mutex_unlock();
      return TILEDB_SM_ERR;
    }
  }

  // Unlock the mutex of the array
  if(open_array->mutex_unlock() != TILEDB_UT_OK) { 
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR;
  }

  // Success 
  return TILEDB_SM_OK;
}

int StorageManager::array_store_schema(
    const std::string& dir, 
    const ArraySchema* array_schema) const {
  // Array schema file
  std::string filename = fs_->append_paths(dir, TILEDB_ARRAY_SCHEMA_FILENAME);
  if (is_file(fs_, filename) && delete_file(fs_, filename) == TILEDB_UT_ERR) {
    std::string errmsg = 
        std::string("Cannot store schema as existing file cannot be deleted");
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Serialize array schema
  void* array_schema_bin;
  size_t array_schema_bin_size;
  if(array_schema->serialize(array_schema_bin, array_schema_bin_size) !=
     TILEDB_AS_OK) {
    tiledb_sm_errmsg = tiledb_as_errmsg;
    return TILEDB_SM_ERR;
  }

  // Store the array schema
  int rc = write_to_file(fs_, filename, array_schema_bin, array_schema_bin_size);
  if (!rc) {
    rc = close_file(fs_, filename);
  }
  if (rc) {
    free(array_schema_bin);
    std::string errmsg = std::string("Cannot store schema");
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Clean up
  free(array_schema_bin);

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::config_set(StorageManagerConfig* config) {
  // Store config locally
  config_ = config;
  fs_ = config_->get_filesystem();

  // Success
  return TILEDB_SM_OK;
} 

int StorageManager::consolidation_filelock_create(
    const std::string& dir) const {
  
  // Create file
  std::string filename = fs_->append_paths(dir, TILEDB_SM_CONSOLIDATION_FILELOCK_NAME);
  if (create_file(fs_, filename, O_WRONLY | O_CREAT | O_SYNC, S_IRWXU) == TILEDB_UT_ERR) {
    std::string errmsg = std::string("Cannot create consolidation filelock");
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::consolidation_filelock_lock(
    const std::string& array_name,
    int& fd, 
    int lock_type) const {
  if (!fs_->locking_support()) {
    return TILEDB_SM_OK;
  }

  // Prepare the flock struct
  struct flock fl;
  if(lock_type == TILEDB_SM_SHARED_LOCK) {
    fl.l_type = F_RDLCK;
  } else if(lock_type == TILEDB_SM_EXCLUSIVE_LOCK) {
    fl.l_type = F_WRLCK;
  } else {
    std::string errmsg = 
        "Cannot lock consolidation filelock; Invalid lock type";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }
  fl.l_whence = SEEK_SET;
  fl.l_start = 0;
  fl.l_len = 0;
  fl.l_pid = getpid();

  // Prepare the filelock name
  std::string array_name_real = real_dir(fs_, array_name);
  std::string filename = fs_->append_paths(array_name_real, TILEDB_SM_CONSOLIDATION_FILELOCK_NAME);

  // Create consolidation lock file if necessary
  if (!fs_->is_file(filename)) {
    if (consolidation_filelock_create(array_name_real)) {
      std::string errmsg =
        std::string("Cannot lock consolidation filelock; consolidation lock file doesn't exist and ")
        + " cannot create consolidation lock file "+filename;
      PRINT_ERROR(errmsg);
      tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
      return TILEDB_SM_ERR;
    }
  }

  // Open the file
  fd = ::open(filename.c_str(), (lock_type == TILEDB_SM_SHARED_LOCK) ? O_RDONLY : O_RDWR);
  if(fd == -1) {
    std::string errmsg = 
        "Cannot lock consolidation filelock; Cannot open filelock";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  } 

  // Acquire the lock
  if(fcntl(fd, F_SETLKW, &fl) == -1) {
    std::string errmsg = "Cannot lock consolidation filelock; Cannot lock";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::consolidation_filelock_unlock(int fd) const {
  if (!fs_->locking_support()) {
    return TILEDB_SM_OK;
  }

  if(::close(fd) == -1) {
    std::string errmsg = 
        "Cannot unlock consolidation filelock; Cannot close filelock";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  } else {
    return TILEDB_SM_OK;
  }
}

int StorageManager::consolidation_finalize(
    Fragment* new_fragment,
    const std::vector<std::string>& old_fragment_names) {
  // Trivial case - there was no consolidation
  if(old_fragment_names.size() == 0)
    return TILEDB_SM_OK;

  // Acquire exclusive lock on consolidation filelock
  int fd;
  if(consolidation_filelock_lock(
      new_fragment->array()->get_array_path_used(),
      fd,
      TILEDB_SM_EXCLUSIVE_LOCK) != TILEDB_SM_OK) {
    delete new_fragment;
    return TILEDB_SM_ERR;
  }

  // Finalize new fragment - makes the new fragment visible to new reads
  int rc = new_fragment->finalize(); 
  delete new_fragment;
  if(rc != TILEDB_FG_OK) { 
    tiledb_sm_errmsg = tiledb_fg_errmsg;
    return TILEDB_SM_ERR;
  }

  // Make old fragments invisible to new reads
  int fragment_num = old_fragment_names.size();
  for(int i=0; i<fragment_num; ++i) {
    // Delete special fragment file inside the fragment directory
    std::string old_fragment_filename = fs_->append_paths(old_fragment_names[i], TILEDB_FRAGMENT_FILENAME);
    if(delete_file(fs_, old_fragment_filename)) {
      std::string errmsg = 
          std::string("Cannot remove fragment file during "
          "finalizing consolidation; ") + strerror(errno);
      PRINT_ERROR(errmsg);
      tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
      return TILEDB_SM_ERR;
    }
  }

  // Unlock consolidation filelock       
  if(consolidation_filelock_unlock(fd) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;


  // Success
  return TILEDB_SM_OK;
}

int StorageManager::create_group_file(const std::string& group) const {
  // Create file
  std::string filename = fs_->append_paths(group, TILEDB_GROUP_FILENAME);
  if(create_file(fs_, filename,  O_WRONLY | O_CREAT | O_SYNC, S_IRWXU) == TILEDB_FS_ERR) {
    std::string errmsg = std::string("Failed to create group file");
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::create_workspace_file(const std::string& workspace) const {
  // Create file
  std::string filename = fs_->append_paths(workspace, TILEDB_WORKSPACE_FILENAME);
  if(create_file(fs_, filename,  O_WRONLY | O_CREAT | O_SYNC, S_IRWXU) == TILEDB_UT_ERR) {
    std::string errmsg = std::string("Failed to create workspace file; ");
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::group_clear(
    const std::string& group) const {
  // Get real group path
  std::string group_real = ::real_dir(fs_, group); 

  // Check if group exists
  if(!is_group(fs_, group_real)) {
    std::string errmsg = 
        std::string("Group '") + group_real + "' does not exist";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Do not delete if it is a workspace
  if(is_workspace(fs_, group_real)) {
    std::string errmsg = 
        std::string("Group '") + group_real + "' is also a workspace";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Delete all groups, arrays and metadata inside the group directory
  std::vector<std::string> all_dirs = ::get_dirs(fs_, group_real);
  for(auto const& dir: all_dirs) {
    if(is_group(fs_, dir)) {
      group_delete(dir);
    } else if(is_metadata(fs_, dir)) {
      metadata_delete(dir);
    } else if(is_array(fs_, dir)){
      array_delete(dir);
    } else {                            // Non TileDB related
      std::string errmsg = 
          std::string("Cannot delete non TileDB related element '") +
          dir + "'";
      PRINT_ERROR(errmsg);
      tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
      return TILEDB_SM_ERR;
    }
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::group_delete(
    const std::string& group) const {
  // Clear the group
  if(group_clear(group) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Delete group directory
  if(delete_dir(fs_, group) != TILEDB_UT_OK) {
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR; 
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::group_move(
    const std::string& old_group,
    const std::string& new_group) const {
  // Get real group directory names
  std::string old_group_real = real_dir(fs_, old_group);
  std::string new_group_real = real_dir(fs_, new_group);

  // Check if the old group is also a workspace
  if(is_workspace(fs_, old_group_real)) {
    std::string errmsg = 
        std::string("Group '") + old_group_real + 
        "' is also a workspace";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Check if the old group exists
  if(!is_group(fs_, old_group_real)) {
    std::string errmsg = 
        std::string("Group '") + old_group_real + 
        "' does not exist";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Make sure that the new group is not an existing directory
  if(is_dir(fs_, new_group_real)) {
    std::string errmsg = 
        std::string("Directory '") + 
        new_group_real + "' already exists";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Check if the new group is inside a workspace or group
  std::string new_group_parent_folder = parent_dir(fs_, new_group_real);
  if(!is_group(fs_, new_group_parent_folder) && 
     !is_workspace(fs_, new_group_parent_folder)) {
    std::string errmsg = 
        std::string("Folder '") + new_group_parent_folder + "' must "
        "be either a workspace or a group";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Rename
  if(move_path(fs_, old_group_real, new_group_real)) {
    std::string errmsg = 
        std::string("Cannot move group; ") + 
        strerror(errno);
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::metadata_clear(
    const std::string& metadata) const {
  // Get real metadata directory name
  std::string metadata_real = ::real_dir(fs_, metadata);

  // Check if the metadata exists
  if(!is_metadata(fs_, metadata_real)) {
    std::string errmsg = 
        std::string("Metadata '") + metadata_real + "' do not exist";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  std::vector<std::string> all_dirs = ::get_dirs(fs_, metadata_real);
  for(auto const& dir: all_dirs) {
    if(is_fragment(fs_, dir)) {
      delete_dir(fs_, dir);
    } else {
      std::string errmsg =
          std::string("Cannot delete non TileDB related element '") +
          dir + "'";
      PRINT_ERROR(errmsg);
      tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
      return TILEDB_SM_ERR;
    }
  }

  return TILEDB_SM_OK;
}

int StorageManager::metadata_delete(
    const std::string& metadata) const {
  // Get real metadata directory name
  std::string metadata_real = ::real_dir(fs_, metadata);

  // Clear the metadata
  if(metadata_clear(metadata_real) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Delete metadata directory
  if(delete_dir(fs_, metadata_real) != TILEDB_UT_OK) {
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR; 
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::metadata_move(
    const std::string& old_metadata,
    const std::string& new_metadata) const {
  // Get real metadata directory name
  std::string old_metadata_real = real_dir(fs_, old_metadata);
  std::string new_metadata_real = real_dir(fs_, new_metadata);

  // Check if the old metadata exists
  if(!is_metadata(fs_, old_metadata_real)) {
    std::string errmsg = 
        std::string("Metadata '") + old_metadata_real + 
        "' do not exist";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Make sure that the new metadata is not an existing directory
  if(is_dir(fs_, new_metadata_real)) {
    std::string errmsg = 
        std::string("Directory '") + 
        new_metadata_real + "' already exists";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Check if the new metadata are inside a workspace, group or array
  std::string new_metadata_parent_folder = parent_dir(fs_, new_metadata_real);
  if(!is_group(fs_, new_metadata_parent_folder) && 
     !is_workspace(fs_, new_metadata_parent_folder) &&
     !is_array(fs_, new_metadata_parent_folder)) {
    std::string errmsg = 
        std::string("Folder '") + new_metadata_parent_folder + "' must "
        "be workspace, group or array";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Rename metadata
  if(move_path(fs_, old_metadata_real, new_metadata_real)) {
    std::string errmsg = 
        std::string("Cannot move metadata; ") + strerror(errno);
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Incorporate new name in the array schema
  ArraySchema* array_schema;
  if(array_load_schema(new_metadata_real.c_str(), array_schema) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;
  array_schema->set_array_name(new_metadata_real.c_str());

  // Store the new schema
  if(array_store_schema(new_metadata_real, array_schema) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Clean up
  delete array_schema;

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::open_array_mtx_destroy() {
#ifdef HAVE_OPENMP
  int rc_omp_mtx = ::mutex_destroy(&open_array_omp_mtx_);
#else
  int rc_omp_mtx = TILEDB_UT_OK;
#endif
  int rc_pthread_mtx = ::mutex_destroy(&open_array_pthread_mtx_);

  // Errors
  if(rc_pthread_mtx != TILEDB_UT_OK || rc_omp_mtx != TILEDB_UT_OK) {
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::open_array_mtx_init() {
#ifdef HAVE_OPENMP
  int rc_omp_mtx = ::mutex_init(&open_array_omp_mtx_);
#else
  int rc_omp_mtx = TILEDB_UT_OK;
#endif
  int rc_pthread_mtx = ::mutex_init(&open_array_pthread_mtx_);

  // Errors
  if(rc_pthread_mtx != TILEDB_UT_OK || rc_omp_mtx != TILEDB_UT_OK) {
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::open_array_mtx_lock() {
#ifdef HAVE_OPENMP
  int rc_omp_mtx = ::mutex_lock(&open_array_omp_mtx_);
#else
  int rc_omp_mtx = TILEDB_UT_OK;
#endif
  int rc_pthread_mtx = ::mutex_lock(&open_array_pthread_mtx_);

  // Errors
  if(rc_pthread_mtx != TILEDB_UT_OK || rc_omp_mtx != TILEDB_UT_OK) {
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::open_array_mtx_unlock() {
#ifdef HAVE_OPENMP
  int rc_omp_mtx = ::mutex_unlock(&open_array_omp_mtx_);
#else
  int rc_omp_mtx = TILEDB_UT_OK;
#endif
  int rc_pthread_mtx = ::mutex_unlock(&open_array_pthread_mtx_);

  // Errors
  if(rc_pthread_mtx != TILEDB_UT_OK || rc_omp_mtx != TILEDB_UT_OK) {
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

void StorageManager::sort_fragment_names(
    std::vector<std::string>& fragment_names) const {
  // Initializations
  int fragment_num = fragment_names.size();
  std::string t_str;
  int64_t stripped_fragment_name_size, t;
  std::vector<std::pair<int64_t, int> > t_pos_vec;
  t_pos_vec.resize(fragment_num);

  // Get the timestamp for each fragment
  for(int i=0; i<fragment_num; ++i) {
    // Strip fragment name
    std::string& fragment_name = fragment_names[i];
    std::string parent_fragment_name = parent_dir(fs_, fragment_name);
    std::string stripped_fragment_name = 
        fragment_name.substr(parent_fragment_name.size() + 1);
    assert(starts_with(stripped_fragment_name, "__"));
    stripped_fragment_name_size = stripped_fragment_name.size();

    // Search for the timestamp in the end of the name after '_'
    for(int j=2; j<stripped_fragment_name_size; ++j) {
      if(stripped_fragment_name[j] == '_') {
        t_str = stripped_fragment_name.substr(
                    j+1,stripped_fragment_name_size-j);
        sscanf(t_str.c_str(), "%lld", (long long int*)&t); 
        t_pos_vec[i] = std::pair<int64_t, int>(t, i);
        break;
      }
   }
  }

  // Sort the names based on the timestamps
  SORT(t_pos_vec.begin(), t_pos_vec.end()); 
  std::vector<std::string> fragment_names_sorted; 
  fragment_names_sorted.resize(fragment_num);
  for(int i=0; i<fragment_num; ++i) 
    fragment_names_sorted[i] = fragment_names[t_pos_vec[i].second];
  fragment_names = fragment_names_sorted;
}

int StorageManager::workspace_clear(const std::string& workspace) const {
  // Get real workspace path
  std::string workspace_real = real_dir(fs_, workspace);

  // Delete all groups, arrays and metadata inside the workspace directory
  std::vector<std::string> all_dirs = ::get_dirs(fs_, workspace_real);
  for(auto const& dir: all_dirs) {
    if(is_group(fs_, dir)) {
      group_delete(dir);
    } else if(is_metadata(fs_, dir)) {
      metadata_delete(dir);
    } else if(is_array(fs_, dir)){
      array_delete(dir);
    } else {                            // Non TileDB related
      std::string errmsg = 
          std::string("Cannot delete non TileDB related element '") +
          dir + "'";
      PRINT_ERROR(errmsg);
      tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
      return TILEDB_SM_ERR;
    }
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::workspace_delete(
    const std::string& workspace) { 
  // Get real paths
  std::string workspace_real = real_dir(fs_, workspace);

  // Check if workspace exists
  if(!is_workspace(fs_, workspace_real)) {
    std::string errmsg = 
        std::string("Workspace '") + workspace_real + "' does not exist";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }  

  // Clear workspace 
  if(workspace_clear(workspace_real) != TILEDB_SM_OK)
    return TILEDB_SM_ERR;

  // Delete directory
  if(delete_dir(fs_, workspace_real) != TILEDB_UT_OK) {
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::workspace_move(
    const std::string& old_workspace, 
    const std::string& new_workspace) {
  // Get real paths
  std::string old_workspace_real = real_dir(fs_, old_workspace);
  std::string new_workspace_real = real_dir(fs_, new_workspace);

  // Check if old workspace exists
  if(!is_workspace(fs_, old_workspace_real)) {
    std::string errmsg = 
        std::string("Workspace '") + old_workspace_real + "' does not exist";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }  

  // Check new workspace
  if(new_workspace_real == "") {
    std::string errmsg = 
        std::string("Invalid workspace '") + new_workspace_real + "'";
    PRINT_ERROR(errmsg); 
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }
  if(is_dir(fs_, new_workspace_real)) {
    std::string errmsg = 
        std::string("Directory '") + new_workspace_real + "' already exists";
    PRINT_ERROR(errmsg); 
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // New workspace should not be inside another workspace, group array or 
  // metadata
  std::string new_workspace_real_parent = parent_dir(fs_, new_workspace_real);
  if(is_workspace(fs_, new_workspace_real_parent) ||
     is_group(fs_, new_workspace_real_parent) ||
     is_array(fs_, new_workspace_real_parent) ||
     is_metadata(fs_, new_workspace_real_parent)) {
    std::string errmsg = 
        std::string("Folder '") + new_workspace_real_parent + 
        "' should not be a workspace, group, array, or metadata";
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Rename directory 
  if(move_path(fs_, old_workspace_real, new_workspace_real)) {
    std::string errmsg = 
        std::string("Cannot move group; ") + strerror(errno);
    PRINT_ERROR(errmsg);
    tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::OpenArray::mutex_destroy() {
#ifdef HAVE_OPENMP
  int rc_omp_mtx = ::mutex_destroy(&omp_mtx_);
#else
  int rc_omp_mtx = TILEDB_UT_OK;
#endif
  int rc_pthread_mtx = ::mutex_destroy(&pthread_mtx_);

  // Error
  if(rc_pthread_mtx != TILEDB_UT_OK || rc_omp_mtx != TILEDB_UT_OK) {
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::OpenArray::mutex_init() {
#ifdef HAVE_OPENMP
  int rc_omp_mtx = ::mutex_init(&omp_mtx_);
#else
  int rc_omp_mtx = TILEDB_UT_OK;
#endif
  int rc_pthread_mtx =  ::mutex_init(&pthread_mtx_);

  // Error
  if(rc_pthread_mtx != TILEDB_UT_OK || rc_omp_mtx != TILEDB_UT_OK) {
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::OpenArray::mutex_lock() {
#ifdef HAVE_OPENMP
  int rc_omp_mtx = ::mutex_lock(&omp_mtx_);
#else
  int rc_omp_mtx = TILEDB_UT_OK;
#endif
  int rc_pthread_mtx = ::mutex_lock(&pthread_mtx_);

  // Error
  if(rc_pthread_mtx != TILEDB_UT_OK || rc_omp_mtx != TILEDB_UT_OK) {
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}

int StorageManager::OpenArray::mutex_unlock() {
#ifdef HAVE_OPENMP
  int rc_omp_mtx = ::mutex_unlock(&omp_mtx_);
#else
  int rc_omp_mtx = TILEDB_UT_OK;
#endif
  int rc_pthread_mtx = ::mutex_unlock(&pthread_mtx_);

  // Error
  if(rc_pthread_mtx != TILEDB_UT_OK || rc_omp_mtx != TILEDB_UT_OK) {
    tiledb_sm_errmsg = tiledb_ut_errmsg;
    return TILEDB_SM_ERR;
  }

  // Success
  return TILEDB_SM_OK;
}
