<file_sep>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Apr  9 11:38:55 2019

@author: <NAME>
"""

import numpy as np
import rdkit
from rdkit import Chem
from rdkit.Chem import Descriptors
import pandas as pd


def generate(smiles, verbose=False):

    moldata= []
    for elem in smiles:
        mol=Chem.MolFromSmiles(elem) 
        moldata.append(mol)
       
    baseData= np.arange(1,1)
    i=0  
    for mol in moldata:        
       
        desc_MolWt = Descriptors.MolWt(mol)
        desc_MolLogP = Descriptors.MolLogP(mol)
        desc_MolMR = Descriptors.MolMR(mol)
        desc_HeavyAtomCount = Descriptors.HeavyAtomCount(mol)
        desc_NumHAcceptors = Descriptors.NumHAcceptors(mol)
        desc_NumHDonors = Descriptors.NumHDonors(mol)
        desc_NumHeteroatoms = Descriptors.NumHeteroatoms(mol)
        desc_NumRotatableBonds = Descriptors.NumRotatableBonds(mol)
        desc_NumValenceElectrons = Descriptors.NumValenceElectrons(mol)              
        desc_NumAromaticRings = Descriptors.NumAromaticRings(mol)        
        desc_NumSaturatedRings = Descriptors.NumSaturatedRings(mol)        
        desc_NumAliphaticRings = Descriptors.NumAliphaticRings(mol)
        desc_RingCount = Descriptors.RingCount(mol)        
        desc_TPSA = Descriptors.TPSA(mol)
        desc_LabuteASA = Descriptors.LabuteASA(mol)        
        desc_BalabanJ = Descriptors.BalabanJ(mol)
        desc_BertzCT = Descriptors.BertzCT(mol) 
                
        row = np.array([desc_MolWt,desc_MolLogP,desc_MolMR,desc_HeavyAtomCount,desc_NumHAcceptors,desc_NumHDonors,desc_NumHeteroatoms,
                        desc_NumRotatableBonds,desc_NumValenceElectrons,desc_NumAromaticRings,desc_NumSaturatedRings,
                        desc_NumAliphaticRings,desc_RingCount,desc_TPSA,desc_LabuteASA,desc_BalabanJ,desc_BertzCT])   
    
        if(i==0):
            baseData=row
        else:
            baseData=np.vstack([baseData, row])
        i=i+1      
    
    columnNames=["MolWt","MolLogP","MolMR","HeavyAtomCount","NumHAcceptors","NumHDonors","NumHeteroatoms",
                 "NumRotatableBonds","NumValenceElectrons","NumAromaticRings","NumSaturatedRings",
                 "NumAliphaticRings","RingCount","TPSA","LabuteASA","BalabanJ","BertzCT"]   
    descriptors = pd.DataFrame(data=baseData,columns=columnNames)
    
    print("Total descriptors generated: 17 x "+str(len(smiles)))
    return descriptors<file_sep># Aqueous Solubility Data Curation

AqSolDB is created by the Autonomous Energy Materials Discovery [AMD] research group, consists of aqueous solubility values of 9,982 unique compounds curated from 9 different publicly available aqueous solubility datasets. This openly accessible dataset, which is the largest of its kind, and will not only serve as a useful reference source of measured solubility data, but also as a much improved and generalizable training data source for building data-driven models.


## Citation

If you use AqSolDB in your study, please cite the following paper.

Paper: [Nature Scientific Data](https://doi.org/10.1038/s41597-019-0151-1) - https://doi.org/10.1038/s41597-019-0151-1


## Other Platforms 

> Online reproducible code: [Code Ocean](https://doi.org/10.24433/CO.1992938.v1)

> Data science kernels: [Kaggle](https://www.kaggle.com/sorkun/aqsoldb-a-curated-aqueous-solubility-dataset)

> Easy search from AqSolDB: [AMD Website](https://www.amdlab.nl/database/AqSolDB/)

> Data Repository: [Harvard Dataverse](https://doi.org/10.7910/DVN/OVHAW8) 


## Overview

This repository has been developed in order to curate various aqueous solubility datasets into a broad and extensive dataset called AqSolDB.

The curation process in this work can be accomplished by executing two python scripts in the given sequence:
1. data-preprocess.py  - for pre-processing the raw data set to a standardized format
2. data-curation.py - for merging the standardized datasets, assigning reliability lables and adding 2D descriptors

These two python scripts call upon functions from other python modules that are defined in:
- preprocess.py
- merge.py
- descriptors.py

Further information about curation process can be found in the associated manuscript.

# Examples 

## data-preprocess.py 

This file converts 2 example sub-datasets (25 instances from raw forms of dataset-A[1] and dataset-H[6]) which are then converted into a standardized format. (This is an example how to preprocess datasets. The preporcessed data files already in the data folder.)

**inputs:**
1. raw-dataset-A.csv (various solubility metrics (g/L, mg/L..) with Name and CAS Number)
2. raw-dataset-H.csv (has solubility values(LogS) with SLN representations) 

**outputs:**
1. dataset-A.csv 
2. dataset-H.csv 

### Note

To apply this method to your own dataset, perform the following steps:
1. Check the available properties, representations, and solubility units of your dataset
2. Select the suitable preprocessing methods from the "preprocess.py" module.

## data-curation.py 

This file curates, i.e., merges datasets, selects most reliable values among multiple occurences, and adds 2D descriptors from 9 different standardized datasets that are obtained after the pre-processing step. 

**inputs:**
1. dataset-A.csv [1]
2. dataset-B.csv [2]
3. dataset-C.csv [3]
4. dataset-D.csv [4]
5. dataset-E.csv [5]
6. dataset-F.csv [6]
7. dataset-G.csv [7]
8. dataset-H.csv [6]
9. dataset-I.csv [8]

**outputs:**
1. dataset_curated.csv

### Note

To apply this method, your input dataset should be in the standardized format (output of preprocessing) having following columns:
- ID
- Name
- InChI
- InChIKey
- SMILES
- Solubility
- Prediction

# License and Copyright
MIT License

Copyright (c) 2019 <NAME>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.



# References

[1] eChemPortal - The Global Portal to Information on Chemical Substances. https://www.echemportal.org/.

[2] <NAME>.  Preliminary Report:  Water Solubility Estimation by Base Compound Modification.Environmental Science Center, Syracuse, NY (1995).

[3] <NAME>., <NAME>., <NAME>., <NAME>.& <NAME>. Calculation of aqueous solubility of crystalline un-ionized organic chemicals and drugs based on structural similarity and physicochemical descriptors.Journal of Chemical Information and Computer Sciences 54, 683–691 (2014).

[4] <NAME>., <NAME>.  Upgrade of PCGEMS Water Solubility Estimation Method. Environmental Science Center, Syracuse, NY(1994)

[5] <NAME>. Estimation of aqueous solubility for a diverse set of organic compounds based on molecular topology.Journal of Chemical Informationand Computer Sciences 40, 773–777 (2000).

[6] <NAME>., <NAME>. & <NAME>. Aqueous solubility prediction based on weighted atom type counts and solvent accessible surface areas. Journal of Chemical Information and Modeling 49, 571–581 (2009).

[7] <NAME>. S. ESOL: estimating aqueous solubility directly from molecular structure. Journal of Chemical Information and Computer Sciences 44,1000–1005 (2004).

[8] <NAME>., <NAME>. & <NAME>. Solubility challenge: can you predict solubilities of 32 molecules using a database of 100 reliable measurements?.Journal of Chemical Information and Modeling 48, 1289–1303 (2008).
<file_sep>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Apr  8 11:31:39 2019

@author: <NAME>
"""

import pandas as pd
import merge
import descriptors


data_a = pd.read_csv('data/dataset-A.csv', header=0) 
data_b = pd.read_csv('data/dataset-B.csv', header=0) 
data_c = pd.read_csv('data/dataset-C.csv', header=0) 
data_d = pd.read_csv('data/dataset-D.csv', header=0) 
data_e = pd.read_csv('data/dataset-E.csv', header=0) 
data_f = pd.read_csv('data/dataset-F.csv', header=0) 
data_g = pd.read_csv('data/dataset-G.csv', header=0) 
data_h = pd.read_csv('data/dataset-H.csv', header=0) 
data_i = pd.read_csv('data/dataset-I.csv', header=0) 


# first merge data directly    
data_merged= pd.concat([data_a,data_b,data_c,data_d,data_e,data_f,data_g,data_h,data_i])
data_merged=data_merged.reset_index(drop=True)

# curate the data    
data_curated = merge.curate(data_merged)


# generate the descriptors    
descriptors = descriptors.generate(data_curated['SMILES'].values)

# combine descriptors with 
data_curated=pd.concat([data_curated, descriptors], axis=1)

# write into CSV file
data_curated.to_csv("results/data_curated.csv", sep=',', encoding='utf-8', index=False)


<file_sep>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Apr  9 11:30:41 2019

@author: <NAME>
"""

import numpy as np
import pandas as pd
import preprocess


def main():
    
    data_path_a="data/raw-dataset-A.csv"
    data_path_h="data/raw-dataset-H.csv"
    
#raw dataset various solubility metrics (g/L, mg/L..) with Name and CAS Number 
    preprocess_dataset_a(data_path_a)
    
#raw dataset has solubility values(LogS) with SLN representations    
    preprocess_dataset_h(data_path_h)





def preprocess_dataset_a(data_path):

    print("\n\n**\nPreprocess of raw-dataset-A is started\n**")
    data = pd.read_csv(data_path, header=0)     
    smiles_list=preprocess.collect_smiles_from_web(data['Name'].values,data['CAS'].values,verbose=0)                
    
    inchi_list,inchikey_list=preprocess.smiles_to_inchi_inchikey(smiles_list,verbose=0)
    id_list=preprocess.generate_id_list("A",len(smiles_list))
    name_list=preprocess.collect_names_from_web(inchikey_list,smiles_list,verbose=0)
    logs_list=preprocess.convert_to_logs(smiles_list,data['Metric'].values,data['OriginalSol'].values,verbose=0)
    prediction_list=preprocess.collect_predictions_from_web(smiles_list,verbose=0)
    
    dataset_a_df = pd.DataFrame(np.column_stack([ id_list, name_list, inchi_list, inchikey_list, smiles_list, logs_list, prediction_list]), 
                                   columns=[ 'ID', 'Name', 'InChI', 'InChIKey', 'SMILES', 'Solubility', 'Prediction'])
    
    
    #filter dataset by removing missing information. (for strings: "XXX" and for numeric: "999")
    dataset_a_df_clean = preprocess.clean(dataset_a_df) 
    
    
    #update ID after filtering
    id_list=preprocess.generate_id_list("A",len(dataset_a_df_clean.index))
    id_clean_df = pd.DataFrame({'ID': id_list})
    dataset_a_df_clean.update(id_clean_df)
    
    #write dataset into CSV file
    dataset_a_df_clean.to_csv('../results/dataset-A.csv', index=False)
    print("**\nPreprocessed dataset-A is written into dataset-A.csv\n**")
    
    return



def preprocess_dataset_h(data_path):

    
    print("\n\n**\nPreprocess of raw-dataset-H is started\n**")
    
    
    data = pd.read_csv(data_path, header=0)     
    smiles_list=preprocess.sln_to_smiles(data['SLN'].values,verbose=0)              
    
    inchi_list,inchikey_list=preprocess.smiles_to_inchi_inchikey(smiles_list,verbose=0)
    id_list=preprocess.generate_id_list("H",len(smiles_list))
    name_list=preprocess.collect_names_from_web(inchikey_list,smiles_list,verbose=0)
    logs_list=data['Solubility'].values
    prediction_list=preprocess.collect_predictions_from_web(smiles_list,verbose=0)
    
    dataset_h_df = pd.DataFrame(np.column_stack([ id_list, name_list, inchi_list, inchikey_list, smiles_list, logs_list, prediction_list]), 
                                   columns=[ 'ID', 'Name', 'InChI', 'InChIKey', 'SMILES', 'Solubility', 'Prediction'])
    
    
    #filter dataset by removing missing information. (for strings: "XXX" and for numeric: "999")
    dataset_h_df_clean = preprocess.clean(dataset_h_df) 
    
    
    #update ID after filtering
    id_list=preprocess.generate_id_list("H",len(dataset_h_df_clean.index))
    id_clean_df = pd.DataFrame({'ID': id_list})
    dataset_h_df_clean.update(id_clean_df)
    
    #write dataset into CSV file
    dataset_h_df_clean.to_csv('../results/dataset-H.csv', index=False)
    print("**\nPreprocessed dataset-H is written into dataset-H.csv\n**")
    
    return


if __name__== "__main__":
    main()<file_sep>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Apr  8 13:52:29 2019

@author: <NAME>
"""

import numpy as np
import pandas as pd
import math


# Selects most reliable values among the multiple occurences
def curate(data_merged):

    smiles_list= data_merged['SMILES'].tolist()
    solubility_list= data_merged['Solubility'].tolist()
    id_list= data_merged['ID'].tolist()
    inchi_list= data_merged['InChI'].tolist()
    name_list= data_merged['Name'].tolist()
    prediction_list= data_merged['Prediction'].tolist()    


# define variables and assign default values    
    dif_val_list = []
    dif_sol_list = []
    
    same_value_counter=0  #same molecules with same values
    different_value_counter_2times=0 #same molecules with different values (2 occurences)
    different_value_counter_mutiple=0 #same molecules with different values (more than 2 occurences)
    
    ocurrence_count=[-999]*len(id_list)
    SD=[-999]*len(id_list)
    reliability_group=["-"]*len(id_list)
    selected_list=[0]*len(id_list)


# First step: Remove same molecules with same solubility values (change their SMILES into "XXX")
    for i in range(0,len(id_list)):        
        same_value_List=[]       
        if(smiles_list[i] != "XXX" ):
            same_value_List.append(i)
# collect same molecules with in range of 0.01 solubility value          
            for j in range(i+1,len(id_list)):  
                if(inchi_list[i]==inchi_list[j]):
                    if(math.fabs(solubility_list[i]-solubility_list[j])<=0.01):
                        same_value_List.append(j)

# select the best source according to: 1:name existance 2: size of the dataset (already in ordered according to size)                            
            if(len(same_value_List)>1):
                bestId=same_value_List[0]
                for sameId in same_value_List:
                    if((pd.isnull(name_list[bestId]) or name_list[bestId]=="-") and ( not pd.isnull(name_list[sameId]) and name_list[sameId]!="-")):
                        bestId=sameId
                same_value_List.remove(bestId)
                for sameId in same_value_List:
                    smiles_list[sameId]="XXX"                    
                    same_value_counter=same_value_counter+1                    
    print ("Total removed same molecule with same value: "+str(same_value_counter))
           

# Second step: Select the most reliable solubility value among the same molecules (change unselected SMILES into XXX)    
    for i in range(0,len(id_list)):       
        same_molecule_List=[]       
        
# collect same molecules with different solubility value   
        if(smiles_list[i] != "XXX" and selected_list[i]==0):    
            same_molecule_List.append(i)
            for j in range(i+1,len(id_list)):  
                if(smiles_list[j] != "XXX" and inchi_list[i]==inchi_list[j]):
                    same_molecule_List.append(j)

# if occurrence count=1 (Group:G1)        
            if(len(same_molecule_List)==1):
                selected_list[i]=1
                reliability_group[i]="G1"
                SD[i]=0
                ocurrence_count[i]=1

# if occurrence count = 2  (closest to reference (prediction) method )                  
            elif(len(same_molecule_List)==2):

# calculate difference betweeen values and prediction (tie breaker)            
                diff1=math.fabs(solubility_list[same_molecule_List[0]]-prediction_list[same_molecule_List[0]])   
                diff2=math.fabs(solubility_list[same_molecule_List[1]]-prediction_list[same_molecule_List[1]])

                bestId=same_molecule_List[0]
                if(diff1<=diff2):
                    smiles_list[same_molecule_List[1]]="XXX"
                    different_value_counter_2times=different_value_counter_2times+1
                    bestId=same_molecule_List[0]
                    selected_list[bestId]=1
                else:
                    smiles_list[same_molecule_List[0]]="XXX"
                    different_value_counter_2times=different_value_counter_2times+1 
                    bestId=same_molecule_List[1]
                    selected_list[bestId]=1
                    
# decide reliability group (if SD>0.5 Group:G2, else Group:G3)   
                diff=math.fabs(solubility_list[same_molecule_List[0]]-solubility_list[same_molecule_List[1]]) 
                if(diff>1):                                                
                    reliability_group[bestId]="G2"
                else:
                    reliability_group[bestId]="G3"                    

# store differences and SD and occurrence count            

                
                SD[bestId]=diff/2
                ocurrence_count[bestId]=2               
                                                      
# if occurrence count > 2 (closest to mean method )                        
            elif(len(same_molecule_List)>2):
                total=0
                different_solubility_values_list=[]
                
                for sameId in same_molecule_List:
                    total=total+solubility_list[sameId]
                    different_solubility_values_list.append(solubility_list[sameId])
                    
                mean=total / len(same_molecule_List)    
                bestId=same_molecule_List[0]
                bestDiff=999
                
                for sameId in same_molecule_List:
                    diff=math.fabs(solubility_list[sameId]-mean) 
                    if(diff<bestDiff):
                        bestId=sameId
                        bestDiff=diff
                selected_list[bestId]=1                        
               
                std=np.std(different_solubility_values_list, axis=0)
                SD[bestId]=std
                ocurrence_count[bestId]=len(same_molecule_List)
                
# decide reliability group (if SD>0.5 Group:G4, else Group:G5)                                
                if(std>0.5):                                                
                    reliability_group[bestId]="G4"
                else:
                    reliability_group[bestId]="G5"                 
                
                same_molecule_List.remove(bestId)
                for sameId in same_molecule_List:
                    smiles_list[sameId]="XXX"
                    different_value_counter_mutiple=different_value_counter_mutiple+1                                        

    
# add reliability information to curated dataset and filter duplicates    
    data_merged['SD']=pd.Series(SD)    
    data_merged['Occurrences']=pd.Series(ocurrence_count)
    data_merged['Group']=pd.Series(reliability_group)
    data_merged=data_merged.drop(columns=['Prediction'])
    data_filtered=data_merged[data_merged['Group'] !="-"]
    data_filtered=data_filtered.reset_index(drop=True)    
    
                    
    return data_filtered


<file_sep>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Apr  3 17:23:41 2019

@authors: <NAME>, <NAME>
"""

import numpy as np
from rdkit import Chem
from rdkit.Chem import rdSLNParse
from rdkit.Chem import rdinchi
from rdkit.Chem import Descriptors
import urllib.request as urllib2  
from time import sleep


#Convert SLN to SMILES 
def sln_to_smiles(sln_list,verbose=0):      
    
    error_counter=0
    warning_counter=0
    smiles_list= []
    inchis_from_sln=[]
    inchis_from_smiles=[]

    for elem in sln_list:
        mol=rdSLNParse.MolFromQuerySLN(elem) 

        if(mol!=None):
            smiles=Chem.MolToSmiles(mol)
            smiles_list.append(smiles)
            inchi, retcode, message, logs, aux =rdinchi.MolToInchi(mol)  
            inchis_from_sln.append(inchi)              
            mol=Chem.MolFromSmiles(smiles) 
            
            if(mol!=None):                
                inchi, retcode, message, logs, aux =rdinchi.MolToInchi(mol)
                inchis_from_smiles.append(inchi)  
            else: 
                inchis_from_smiles.append("XXX")
                if(verbose!=0):
                    print(elem+ ": SMILES cannot converted to mol object, added XXX instead!")

        else:        
            smiles_list.append("XXX")
            print(elem+ ": SLN can not converted to mol object, added XXX instead!")
            error_counter=error_counter+1
        
    for i1,i2 in zip(inchis_from_sln,inchis_from_smiles):
        if(i1!=i2):
            if(verbose!=0):
                print("Warning:"+i1+" - "+i2)
            warning_counter=warning_counter+1
     
    print("\nConversion from SLN to SMILES is completed.")    
    print("Total errors:"+str(error_counter))
    print("Total warnings:"+str(warning_counter)+"\n")    
              
    return smiles_list

#Returns InChI InChIKey from SMILES
def smiles_to_inchi_inchikey(smiles,verbose=0):
          
    error_counter=0
    warning_counter=0
    inchis=[]
    inchis2=[]
    inchiKeys=[]

    for elem in smiles:
        mol=Chem.MolFromSmiles(elem) 
        if(mol!=None):
            inchi, retcode, message, logs, aux =rdinchi.MolToInchi(mol)
            inchiKey=rdinchi.InchiToInchiKey(inchi)   
            inchis.append(inchi)  
            inchiKeys.append(inchiKey)   
            try:
                mol, retcode, message, logs=rdinchi.InchiToMol(inchi)
                if(mol!=None):                
                    inchi2, retcode, message, logs, aux =rdinchi.MolToInchi(mol)
                    inchis2.append(inchi2)  
                else: 
                    inchis2.append("XXX") 
                    if(verbose!=0):
                        print(elem+ ": InChI cannot converted to mol object, added XXX instead!")
            except:
                if(verbose!=0):
                    print(retcode)
                    print(message)
                    print("Smiles:"+elem)
                inchis2.append("XXX")  
            
        else:  
            inchis.append("XXX")
            inchiKeys.append("XXX")
            inchis2.append("XXX")            
            if(verbose!=0):
                print(elem+ ": can not converted added XXX instead! ")            
            error_counter=error_counter+1

    for i1,i2 in zip(inchis,inchis2):
        if(i1!=i2):
            if(verbose!=0):
                print("Warning:"+i1+" - "+i2)
            warning_counter=warning_counter+1

    print("\nGeneration of InChI and InChIKey from SMILES is completed.")    
    print("Total errors:"+str(error_counter))
    print("Total warnings:"+str(warning_counter)+"\n")    
                                          
    return inchis,inchiKeys

#Returns ID list by dataset name and size
def generate_id_list(name,size):
          
    id_list= []

    for i in range(1, size+1):
        id_list.append(name+"-"+str(i))
        
    return id_list

#Converts different solubility units into the LogS
def convert_to_logs(smiles_list,unit_list,solubility_list,verbose=0):

    error_counter=0
    logs_list=[]
    
    for i in range(len(smiles_list)):  
        try:            
            mw=Descriptors.MolWt(Chem.MolFromSmiles(smiles_list[i]))                        
            
            try:
                if(unit_list[i]=="g/L"):
                    logs_list.append(np.log10(float(solubility_list[i])/mw))
                
                elif(unit_list[i]=="mg/L"):
                    logs_list.append(np.log10(float(solubility_list[i])/(mw*1000)))
                
                elif(unit_list[i]=="µg/L"):
                    logs_list.append(np.log10(float(solubility_list[i])/(mw*1000000)))
    
                elif(unit_list[i]=="M"):
                    logs_list.append(np.log10(float(solubility_list[i])))
                    
                elif(unit_list[i]=="µM"):
                    logs_list.append(np.log10(float(solubility_list[i])/1000000))
                
                elif(unit_list[i]=="LogS"):
                    logs_list.append(solubility_list[i])
        
                else:
                    if(verbose!=0):
                        print("Unknown metric:"+unit_list[i])  
                    error_counter=error_counter+1
                    logs_list.append(999)
            except:
                if(verbose!=0):
                    print("Error during the calculation! Solubility:" +str(solubility_list[i]) +" unit:"+unit_list[i])      
                error_counter=error_counter+1
                logs_list.append(999)
        except:
            if(verbose!=0):
                print("Error during the creation of mol object! Smiles:" +smiles_list[i])      
            error_counter=error_counter+1
            logs_list.append(999)
            
    print("\nConversion of units into LogS is completed.")   
    print("Total instances:"+str(len(smiles_list)))
    print("Total errors:"+str(error_counter)+"\n")
        
    return logs_list


#Collects SMILES from the Chemical Identifier Resolver web service of the National Cancer Institute using Name and CAS number
def collect_smiles_from_web(name_list,cas_list,verbose=0):

    error_counter=0
    smiles_list = []
       
    for i in range(len(name_list)):
        if(verbose!=0):
            print(name_list[i])
        try:
            sleep(0.04)
            target_url="https://cactus.nci.nih.gov/chemical/structure/"+cas_list[i].lstrip("0").rstrip(" ")+"/smiles"    
            context = urllib2.urlopen(target_url) 
            smiles=context.read().decode('utf-8')               
            smiles_list.append(smiles)      
            
        except:
            if(verbose!=0):
                print("Cannot collected SMILES using CAS:"+cas_list[i])    
                print("Trying to collect using Name:"+name_list[i])  
            try:
                sleep(0.04)
                target_url="https://cactus.nci.nih.gov/chemical/structure/"+name_list[i].rstrip(" ").replace(" ", "%20")+"/smiles"
                context = urllib2.urlopen(target_url) 
                smiles=context.read().decode('utf-8')
                smiles_list.append(smiles)            
            except:     
                if(verbose!=0):
                    print("Cannot collected SMILES using Name:"+name_list[i])
                smiles_list.append("XXX")
                error_counter=error_counter+1
                
    print("\nCollecting SMILES from webservice is completed.")    
    print("Total instances:"+str(len(name_list)))
    print("Total errors:"+str(error_counter)+"\n")
    
    return smiles_list           


#Collects names from the Chemical Identifier Resolver web service of the National Cancer Institute using InChIKey and SMILES
def collect_names_from_web(incikey_list,smiles_list,verbose=0):

    error_counter=0
    name_list = []
       
    for i in range(len(incikey_list)):
        try:
            sleep(0.04)
            target_url="https://cactus.nci.nih.gov/chemical/structure/"+incikey_list[i]+"/iupac_name"    
            context = urllib2.urlopen(target_url) 
            name=context.read().decode('utf-8')               
            name_list.append(name)      
            
        except:
            if(verbose!=0):
                print("Cannot collected name using InChIKey:"+incikey_list[i])    
                print("Trying to collect using SMILES:"+smiles_list[i])  
            try:
                sleep(0.04)
                target_url="https://cactus.nci.nih.gov/chemical/structure/"+smiles_list[i].replace("#","%23")+"/iupac_name"
                context = urllib2.urlopen(target_url) 
                name=context.read().decode('utf-8')
                name_list.append(name)             
            except:     
                if(verbose!=0):
                    print("Cannot collected name using SMILES:"+smiles_list[i])
                name_list.append("XXX")
                error_counter=error_counter+1
                
    print("\nCollecting names from webservice is completed.")    
    print("Total instances:"+str(len(incikey_list)))
    print("Total errors:"+str(error_counter)+"\n")
    
    return name_list     


#Collects solubility predictions from the ALOGPS 2.1 using InChIKey and SMILES
def collect_predictions_from_web(smiles_list,verbose=0):

    error_counter=0
    prediction_list = []
       
    for i in range(len(smiles_list)):
        try:
            sleep(0.04)
            target_url="http://192.168.127.12/web/alogps/calc?SMILES="+smiles_list[i]    
            context = urllib2.urlopen(target_url) 
            text=context.read().decode('utf-8')
            if("could not" not in text):
                prediction=text.split(" ")[5]
                prediction_list.append(prediction)   
            else:
                raise Exception("Cannot collected prediction for:"+smiles_list[i])
            
        except:
            if(verbose!=0):
                print("Cannot collected prediction for:"+smiles_list[i])    
            prediction_list.append(999)
            error_counter=error_counter+1
                
    print("\nCollecting predictions from webservice is completed.")    
    print("Total instances:"+str(len(smiles_list)))
    print("Total errors:"+str(error_counter)+"\n")
    
    return prediction_list   

#filter dataset by removing missing information. (for strings: "XXX" and for numeric: "999")
def clean(dataset,verbose=1):

    dataset_clean = dataset[(dataset.InChIKey != "XXX") & (dataset.Solubility != "999") ].reset_index(drop=True)
    
    print("Total removed during the cleaning:"+str(len(dataset)-len(dataset_clean))+"\n")
    
    return dataset_clean   



