text:
import sys
import os
import datetime
import glob
import shutil
from pmagpy import pmag
from pmagpy import ipmag
from pmagpy import version
from pmagpy import contribution_builder as cb
from programs import thumbnails
VERBOSE = True

def error_log(msg, loc="", program="", con_id=""):
    con_id = str(con_id)
    with open('errors.txt', 'a') as log:
        log.write(con_id + '\t' + str(datetime.datetime.now()) + '\t' + loc + '\t' + program + '\t' + msg + '\n')
    full_msg = '-W- ' + con_id + '\t' + loc + '\t' + program + '\t' + msg + '\n'
    if VERBOSE:
        print(full_msg)
    sys.stderr.write(full_msg)

def info_log(msg, loc="", program=""):
    with open('log.txt', 'a') as log:
        log.write(str(datetime.datetime.now()) + "\t" + loc + "\t" + program + '\t' + msg + '\n')

def check_for_reqd_cols(data, reqd_cols):
    """
    Check data (PmagPy list of dicts) for required columns
    """
    missing = []
    for col in reqd_cols:
        if col not in data[0]:
            missing.append(col)
    return missing

def main():
    """
    NAME
        make_magic_plots.py

    DESCRIPTION
        inspects magic directory for available data and makes plots

    SYNTAX
        make_magic_plots.py [command line options]

    INPUT
        magic files

    OPTIONS
        -h prints help message and quits
        -f FILE specifies input file name
        -fmt [png,eps,svg,jpg,pdf] specify format, default is png
    """
    if '-h' in sys.argv:
        print(main.__doc__)
        sys.exit()
    # reset log files
    for fname in ['log.txt', 'errors.txt']:
        f = os.path.join(os.getcwd(), fname)
        if os.path.exists(f):
            os.remove(f)
    image_recs = []
    dirlist = ['./']
    dir_path = os.getcwd()
    #
    if '-fmt' in sys.argv:
        ind = sys.argv.index("-fmt")
        fmt = sys.argv[ind + 1]
    else:
        fmt = 'png'
    do_thumbnails = False if "-no-thumb" in sys.argv else True
    if '-f' in sys.argv:
        ind = sys.argv.index("-f")
        filelist = [sys.argv[ind + 1]]
    else:
        filelist = os.listdir(dir_path)
    ## initialize some variables
    samp_file = 'samples.txt'
    meas_file = 'measurements.txt'
    #loc_key = 'location'
    loc_file = 'locations.txt'
    method_key = 'method_codes'
    dec_key = 'dir_dec'
    inc_key = 'dir_inc'
    tilt_corr_key = "dir_tilt_correction"
    aniso_tilt_corr_key = "aniso_tilt_correction"
    hyst_bcr_key = "hyst_bcr"
    hyst_mr_key = "hyst_mr_moment"
    hyst_ms_key = "hyst_ms_moment"
    hyst_bc_key = "hyst_bc"
    Mkeys = ['magnitude', 'magn_moment', 'magn_volume', 'magn_mass']
    results_file = 'sites.txt'
    hyst_file = 'specimens.txt'
    aniso_file = 'specimens.txt'
    # create contribution and propagate data throughout
    full_con = cb.Contribution()
    full_con.propagate_location_to_measurements()
    full_con.propagate_location_to_specimens()
    full_con.propagate_location_to_samples()
    if not full_con.tables:
        print('-E- No MagIC tables could be found in this directory')
        error_log("No MagIC tables found")
        return
    # try to get the contribution id for error logging
    con_id = ""
    if 'contribution' in full_con.tables:
        if 'id' in full_con.tables['contribution'].df.columns:
            con_id = full_con.tables['contribution'].df.iloc[0]['id']
    # check to see if propagation worked, otherwise you can't plot by location
    lowest_table = None
    for table in full_con.ancestry:
        if table in full_con.tables:
            lowest_table = table
            break

    do_full_directory = False
    # check that locations propagated down to the lowest table in the contribution
    if 'location' in full_con.tables[lowest_table].df.columns:
        if 'locations' not in full_con.tables:
            info_log('location names propagated to {}, but could not be validated'.format(lowest_table))
        # are there any locations in the lowest table?
        elif not all(full_con.tables[lowest_table].df['location'].isnull()):
            locs = full_con.tables['locations'].df.index.unique()
            lowest_locs = full_con.tables[lowest_table].df['location'].unique()
            incorrect_locs = set(lowest_locs).difference(set(locs))
            # are they actual locations?
            if not incorrect_locs:
                info_log('location names propagated to {}'.format(lowest_table))
            else:
                do_full_directory = True
                error_log('location names did not propagate fully to {} table (looks like there are some naming inconsistencies between tables)'.format(lowest_table), con_id=con_id)
        else:
            do_full_directory = True
            error_log('could not propagate location names down to {} table'.format(lowest_table), con_id=con_id)
    else:
        do_full_directory = True
        error_log('could not propagate location names down to {} table'.format(lowest_table), con_id=con_id)

    all_data = {}
    all_data['measurements'] = full_con.tables.get('measurements', None)
    all_data['specimens'] = full_con.tables.get('specimens', None)
    all_data['samples'] = full_con.tables.get('samples', None)
    all_data['sites'] = full_con.tables.get('sites', None)
    all_data['locations'] = full_con.tables.get('locations', None)
    if 'locations' in full_con.tables:
        locations = full_con.tables['locations'].df.index.unique()
    else:
        locations = ['']
    dirlist = [loc for loc in locations if cb.not_null(loc, False) and loc != 'nan']
    if not dirlist:
        dirlist = ["./"]
    if do_full_directory:
        dirlist = ["./"]

    # plot the whole contribution as one location
    if dirlist == ["./"]:
        error_log('plotting the entire contribution as one location', con_id=con_id)
        for fname in os.listdir("."):
            if fname.endswith(".txt"):
                shutil.copy(fname, "tmp_" + fname)

    # if possible, go through all data by location
    # use tmp_*.txt files to separate out by location

    for loc in dirlist:
        print('\nworking on: ', loc)

        def get_data(dtype, loc_name):
            """
            Extract data of type dtype for location loc_name.
            Write tmp_dtype.txt files if possible.
            """
            if cb.not_null(all_data[dtype], False):
                data_container = all_data[dtype]
                if loc_name == "./":
                    data_df = data_container.df
                else:
                    # awkward workaround for chars like "(" and "?" that break in regex
                    try:
                        data_df = data_container.df[data_container.df['location'].astype(str).str.contains(loc_name, na=False)]
                    except: #sre_constants.error:
                        data_df = data_container.df[data_container.df['location'] == loc_name]

                data = data_container.convert_to_pmag_data_list(df=data_df)
                res = data_container.write_magic_file('tmp_{}.txt'.format(dtype), df=data_df)
                if not res:
                    return [], []
                return data, data_df
            return [], []

        meas_data, meas_df = get_data('measurements', loc)
        spec_data, spec_df = get_data('specimens', loc)
        samp_data, samp_df = get_data('samples', loc)
        site_data, site_df = get_data('sites', loc)
        loc_data, loc_df = get_data('locations', loc)

        con = cb.Contribution(read_tables=[])
        con.tables['measurements'] = cb.MagicDataFrame(df=meas_df, dtype="measurements")
        con.tables['specimens'] = cb.MagicDataFrame(df=spec_df, dtype="specimens")
        con.tables['samples'] = cb.MagicDataFrame(df=samp_df, dtype="samples")
        con.tables['sites'] = cb.MagicDataFrame(df=site_df, dtype="sites")
        con.tables['locations'] = cb.MagicDataFrame(df=loc_df, dtype="locations")

        if loc == "./":  # if you can't sort by location, do everything together
            con = full_con
            try:
                meas_data = con.tables['measurements'].convert_to_pmag_data_list()
            except KeyError:
                meas_data = None
            try:
                spec_data = con.tables['specimens'].convert_to_pmag_data_list()
            except KeyError:
                spec_data = None
            try:
                samp_data = con.tables['samples'].convert_to_pmag_data_list()
            except KeyError:
                samp_data = None
            try:
                site_data = con.tables['sites'].convert_to_pmag_data_list()
            except KeyError:
                site_data = None

        crd = 's'
        if 'samples' in con.tables:
            if 'azimuth' in con.tables['samples'].df.columns:
                if any(con.tables['samples'].df['azimuth'].dropna()):
                    crd = 'g'
        if crd == 's':
            print('using specimen coordinates')
        else:
            print('using geographic coordinates')
        if meas_file in filelist and meas_data:  # start with measurement data
            print('working on plotting measurements data')
            data = meas_data
            file_type = 'measurements'
            # looking for  zeq_magic possibilities
            # get all non blank method codes
            AFZrecs = pmag.get_dictitem(data, method_key, 'LT-AF-Z', 'has')
            # get all non blank method codes
            TZrecs = pmag.get_dictitem(data, method_key, 'LT-T-Z', 'has')
            # get all non blank method codes
            MZrecs = pmag.get_dictitem(data, method_key, 'LT-M-Z', 'has')
            # get all dec measurements
            Drecs = pmag.get_dictitem(data, dec_key, '', 'F')
            # get all inc measurements
            Irecs = pmag.get_dictitem(data, inc_key, '', 'F')
            for key in Mkeys:
                Mrecs = pmag.get_dictitem(
                    data, key, '', 'F')  # get intensity data
                if len(Mrecs) > 0:
                    break
            # potential for stepwise demag curves
            if len(AFZrecs) > 0 or len(TZrecs) > 0 or len(MZrecs) > 0 and len(Drecs) > 0 and len(Irecs) > 0 and len(Mrecs) > 0:
                #CMD = 'zeq_magic.py -f tmp_measurements.txt -fsp tmp_specimens.txt -fsa tmp_samples.txt -fsi tmp_sites.txt -sav -fmt ' + fmt + ' -crd ' + crd + " -new"
                CMD = "ipmag.zeq_magic(crd={}, n_plots='all', contribution={}, image_records=True, fmt={})".format(crd, con, fmt)
                print(CMD)
                info_log(CMD, loc)
                res, outfiles, zeq_images = ipmag.zeq_magic(crd=crd, n_plots='all',
                                                            contribution=con, image_records=True, fmt=fmt)
                image_recs.extend(zeq_images)
            # looking for  thellier_magic possibilities
            if len(pmag.get_dictitem(data, method_key, 'LP-PI-TRM', 'has')) > 0:
                #CMD = 'thellier_magic.py -f tmp_measurements.txt -fsp tmp_specimens.txt -sav -fmt ' + fmt
                CMD = "ipmag.thellier_magic(n_specs='all', fmt='{}', contribution={}, image_records=True)".format(fmt, con)
                print(CMD)
                info_log(CMD, loc)
                res, outfiles, thellier_images = ipmag.thellier_magic(n_specs='all', fmt=fmt, contribution=con, image_records=True)
                image_recs.extend(thellier_images)
            # looking for hysteresis possibilities
            if len(pmag.get_dictitem(data, method_key, 'LP-HYS', 'has')) > 0:  # find hyst experiments
                # check for reqd columns
                missing = check_for_reqd_cols(data, ['treat_temp'])
                if missing:
                    error_log('LP-HYS method code present, but required column(s) [{}] missing'.format(", ".join(missing)), loc, "quick_hyst.py", con_id=con_id)
                else:
                    #CMD = 'quick_hyst.py -f tmp_measurements.txt -sav -fmt ' + fmt
                    CMD = "ipmag.quick_hyst(fmt='{}', n_plots='all', contribution={}, image_records=True)".format(fmt, con)
                    print(CMD)
                    info_log(CMD, loc)
                    res, outfiles, quick_hyst_recs = ipmag.quick_hyst(fmt=fmt, n_plots='all', contribution=con, image_records=True)
                    image_recs.extend(quick_hyst_recs)
            # equal area plots of directional data
            # at measurement level (by specimen)
            if data:
                missing = check_for_reqd_cols(data, ['dir_dec', 'dir_inc'])
                if not missing:
                    #CMD = "eqarea_magic.py -f tmp_measurements.txt -obj spc -sav -no-tilt -fmt " + fmt
                    CMD = "ipmag.eqarea_magic(fmt=fmt, n_plots='all', ignore_tilt=True, plot_by='spc', contribution={}, source_table='measurements', image_records=True)".format(fmt, con)
                    print(CMD)
                    info_log(CMD, loc, "eqarea_magic.py")
                    res, outfiles, eqarea_spc_images = ipmag.eqarea_magic(fmt=fmt, n_plots='all',
                                                                          ignore_tilt=True, plot_by="spc",
                                                                          contribution=con,
                                                                          source_table="measurements",
                                                                          image_records=True)
                    image_recs.extend(eqarea_spc_images)

        else:
            if VERBOSE:
                print('-I- No measurement data found')

        # site data
        if results_file in filelist and site_data:
            print('-I- result file found', results_file)
            data = site_data
            file_type = 'sites'
            print('-I- working on site directions')
            print('number of datapoints: ', len(data), loc)
            dec_key = 'dir_dec'
            inc_key = 'dir_inc'
            int_key = 'int_abs'
            SiteDIs = pmag.get_dictitem(data, dec_key, "", 'F')  # find decs
            SiteDIs = pmag.get_dictitem(
                SiteDIs, inc_key, "", 'F')  # find decs and incs
            dir_data_found = len(SiteDIs)
            print('{} Dec/inc pairs found'.format(dir_data_found))
            if SiteDIs:
                # then convert tilt_corr_key to correct format
                old_SiteDIs = SiteDIs
                SiteDIs = []
                for rec in old_SiteDIs:
                    if tilt_corr_key not in rec:
                        rec[tilt_corr_key] = "0"
                    # make sure tilt_corr_key is a correct format
                    try:
                        rec[tilt_corr_key] = str(int(float(rec[tilt_corr_key])))
                    except ValueError:
                        rec[tilt_corr_key] = "0"
                    SiteDIs.append(rec)

                print('number of individual directions: ', len(SiteDIs))
                # tilt corrected coordinates
                SiteDIs_t = pmag.get_dictitem(SiteDIs, tilt_corr_key, '100',
                                              'T', float_to_int=True)
                print('number of tilt corrected directions: ', len(SiteDIs_t))
                SiteDIs_g = pmag.get_dictitem(
                    SiteDIs, tilt_corr_key, '0', 'T', float_to_int=True)  # geographic coordinates
                print('number of geographic  directions: ', len(SiteDIs_g))
                SiteDIs_s = pmag.get_dictitem(
                    SiteDIs, tilt_corr_key, '-1', 'T', float_to_int=True)  # sample coordinates
                print('number of sample  directions: ', len(SiteDIs_s))
                SiteDIs_x = pmag.get_dictitem(
                    SiteDIs, tilt_corr_key, '', 'T')  # no coordinates
                print('number of no coordinates  directions: ', len(SiteDIs_x))
                if len(SiteDIs_t) > 0 or len(SiteDIs_g) > 0 or len(SiteDIs_s) > 0 or len(SiteDIs_x) > 0:
                    CRD = ""
                    if len(SiteDIs_t) > 0:
                        CRD = ' -crd t'
                        crd = "t"
                    elif len(SiteDIs_g) > 0:
                        CRD = ' -crd g'
                        crd = "g"
                    elif len(SiteDIs_s) > 0:
                        CRD = ' -crd s'
                        crd = "s"
                    #CMD = 'eqarea_magic.py -f tmp_sites.txt -fsp tmp_specimens.txt -fsa tmp_samples.txt -flo tmp_locations.txt -sav -fmt ' + fmt + CRD
                    CMD = "ipmag.eqarea_magic(crd={}, fmt={}, n_plots='all', contribution={}, source_table='sites')".format(fmt, crd, con)
                    print(CMD)
                    info_log(CMD, loc)
                    res, outfiles, eqarea_site_recs = ipmag.eqarea_magic(crd=crd, fmt=fmt, n_plots='all',
                                                                   contribution=con, source_table="sites",
                                                                   image_records=True)
                    image_recs.extend(eqarea_site_recs)
                else:
                    if dir_data_found:
                        error_log('{} dec/inc pairs found, but no equal area plots were made'.format(dir_data_found), loc, "equarea_magic.py", con_id=con_id)
            #
            print('-I- working on VGP map')
            VGPs = pmag.get_dictitem(
                SiteDIs, 'vgp_lat', "", 'F')  # are there any VGPs?
            if len(VGPs) > 0:  # YES!
                #CMD = 'vgpmap_magic.py -f tmp_sites.txt -prj moll -res c -sym ro 5 -sav -fmt {}'.format(fmt)
                CMD = "ipmag.vgpmap_magic(proj='moll', sym='ro', size=5, fmt={}, contribution={})".format(fmt, con)
                print(CMD)
                info_log(CMD, loc, 'vgpmap_magic.py')
                res, outfiles, vgpmap_recs = ipmag.vgpmap_magic(proj='moll', sym='ro', size=5,
                                                                fmt=fmt, contribution=con,
                                                                image_records=True)
                image_recs.extend(vgpmap_recs)
            else:
                print('-I- No vgps found')

            print('-I- Look for intensities')
            # is there any intensity data?
            if site_data:
                if int_key in site_data[0].keys():
                    # old way, wasn't working right:
                    #CMD = 'magic_select.py  -key ' + int_key + ' 0. has -F tmp1.txt -f tmp_sites.txt'
                    Selection = pmag.get_dictkey(site_data, int_key, dtype="f")
                    selection = [i * 1e6 for i in Selection if i != 0]
                    loc = loc.replace(" ", "_")
                    if loc == "./":
                        loc_name = ""
                    else:
                        loc_name = loc
                    histfile = 'LO:_' + loc_name + \
                        '_TY:_intensities_histogram:_.' + fmt
                    CMD = "histplot.py -twin -b 1 -xlab 'Intensity (uT)' -sav -f intensities.txt -F " + histfile
                    CMD = "ipmag.histplot(data=selection, outfile=histfile, xlab='Intensity (uT)', binsize=1, norm=-1, save_plots=True)".format(histfile)
                    info_log(CMD, loc)
                    print(CMD)
                    ipmag.histplot(data=selection, outfile=histfile, xlab="Intensity (uT)",
                                   binsize=1, norm=-1, save_plots=True)
                    histplot_rec = {'file': histfile, 'type': 'Other', 'title': 'Intensity histogram',
                                    'software_packages': version.version, 'keywords': "",
                                    'timestamp': datetime.date.today().isoformat()}
                    image_recs.append(histplot_rec)
                else:
                    print('-I- No intensities found')
            else:
                print('-I- No intensities found')

        ##
        if hyst_file in filelist and spec_data:
            print('working on hysteresis', hyst_file)
            data = spec_data
            file_type = 'specimens'
            hdata = pmag.get_dictitem(data, hyst_bcr_key, '', 'F')
            hdata = pmag.get_dictitem(hdata, hyst_mr_key, '', 'F')
            hdata = pmag.get_dictitem(hdata, hyst_ms_key, '', 'F')
            # there are data for a dayplot
            hdata = pmag.get_dictitem(hdata, hyst_bc_key, '', 'F')
            if len(hdata) > 0:
                CMD = "ipmag.dayplot_magic(save=True, fmt={}, contribution={}, image_records=True)".format(fmt, con)
                info_log(CMD, loc)
                print(CMD)
                res, outfiles, dayplot_recs = ipmag.dayplot_magic(save=True, fmt=fmt,
                                                                  contribution=con, image_records=True)
                image_recs.extend(dayplot_recs)
            else:
                print('no hysteresis data found')
        if aniso_file in filelist and spec_data:  # do anisotropy plots if possible
            print('working on anisotropy', aniso_file)
            data = spec_data
            file_type = 'specimens'

            # make sure there is some anisotropy data
            if not data:
                print('No anisotropy data found')
            elif 'aniso_s' not in data[0]:
                print('No anisotropy data found')
            else:
                # get specimen coordinates
                if aniso_tilt_corr_key not in data[0]:
                    sdata = data
                else:
                    sdata = pmag.get_dictitem(
                        data, aniso_tilt_corr_key, '-1', 'T', float_to_int=True)
                # get specimen coordinates
                gdata = pmag.get_dictitem(
                    data, aniso_tilt_corr_key, '0', 'T', float_to_int=True)
                # get specimen coordinates
                tdata = pmag.get_dictitem(
                    data, aniso_tilt_corr_key, '100', 'T', float_to_int=True)
                if len(sdata) > 3:
                    CMD = "ipmag.aniso_magic(iboot=0, ihext=1, crd='s', fmt={}, contribution={})".format(fmt, con)
                    print(CMD)
                    info_log(CMD, loc)
                    res, files, aniso_recs = ipmag.aniso_magic(iboot=0, ihext=1, crd="s", fmt=fmt,
                                                               contribution=con, image_records=True)
                    image_recs.extend(aniso_recs)
                if len(gdata) > 3:
                    CMD = "ipmag.aniso_magic(iboot=0, ihext=1, crd='g', fmt={}, contribution={})".format(fmt, con)
                    print(CMD)
                    info_log(CMD, loc)
                    res, files, aniso_recs = ipmag.aniso_magic(iboot=0, ihext=1, crd="g", fmt=fmt,
                                                               contribution=con, image_records=True)
                    image_recs.extend(aniso_recs)
                if len(tdata) > 3:
                    CMD = "ipmag.aniso_magic(iboot=0, ihext=1, crd='g', fmt={}, contribution={})".format(fmt, con)
                    print(CMD)
                    info_log(CMD, loc)
                    res, files, aniso_recs = ipmag.aniso_magic(iboot=0, ihext=1, crd="t", fmt=fmt,
                                                               contribution=con, image_records=True)
                    image_recs.extend(aniso_recs)

        # remove temporary files
        for fname in glob.glob('tmp*.txt'):
            os.remove(fname)

    # now we need full contribution data
    if loc_file in filelist and loc_data:
        #data, file_type = pmag.magic_read(loc_file)  # read in location data
        data = loc_data
        print('-I- working on pole map')
        poles = pmag.get_dictitem(
            data, 'pole_lat', "", 'F')  # are there any poles?
        poles = pmag.get_dictitem(
            poles, 'pole_lon', "", 'F')  # are there any poles?
        if len(poles) > 0:  # YES!
            CMD = 'polemap_magic.py -sav -fmt {} -rev gv 40'.format(fmt)
            CMD =  'ipmag.polemap_magic(flip=True, rsym="gv", rsymsize=40, fmt="{}", contribution={})'.format(fmt, full_con)
            print(CMD)
            info_log(CMD, "all locations", "polemap_magic.py")
            res, outfiles, polemap_recs = ipmag.polemap_magic(flip=True, rsym="gv", rsymsize=40,
                                                            fmt=fmt, contribution=full_con,
                                                            image_records=True)
            image_recs.extend(polemap_recs)
        else:
            print('-I- No poles found')

    if image_recs:
        new_image_file = os.path.join(dir_path, 'new_images.txt')
        old_image_file = os.path.join(dir_path, 'images.txt')
        pmag.magic_write(new_image_file, image_recs, 'images')
        if os.path.exists(old_image_file):
            ipmag.combine_magic([old_image_file, new_image_file], outfile=old_image_file,
                                magic_table="images", dir_path=dir_path)
        else:
            os.rename(new_image_file, old_image_file)

    if do_thumbnails:
        thumbnails.make_thumbnails(dir_path)

if __name__ == "__main__":
    main()


text:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Ph4ct3x.App.XamarinForms.Models;

namespace Ph4ct3x.App.XamarinForms.Services
{
    public class MockDataStore : IDataStore<Item>
    {
        readonly List<Item> items;

        public MockDataStore()
        {
            items = new List<Item>()
            {
                new Item { Id = Guid.NewGuid().ToString(), Text = "First item", Description="This is an item description." },
                new Item { Id = Guid.NewGuid().ToString(), Text = "Second item", Description="This is an item description." },
                new Item { Id = Guid.NewGuid().ToString(), Text = "Third item", Description="This is an item description." },
                new Item { Id = Guid.NewGuid().ToString(), Text = "Fourth item", Description="This is an item description." },
                new Item { Id = Guid.NewGuid().ToString(), Text = "Fifth item", Description="This is an item description." },
                new Item { Id = Guid.NewGuid().ToString(), Text = "Sixth item", Description="This is an item description." }
            };
        }

        public async Task<bool> AddItemAsync(Item item)
        {
            items.Add(item);

            return await Task.FromResult(true);
        }

        public async Task<bool> UpdateItemAsync(Item item)
        {
            var oldItem = items.Where((Item arg) => arg.Id == item.Id).FirstOrDefault();
            items.Remove(oldItem);
            items.Add(item);

            return await Task.FromResult(true);
        }

        public async Task<bool> DeleteItemAsync(string id)
        {
            var oldItem = items.Where((Item arg) => arg.Id == id).FirstOrDefault();
            items.Remove(oldItem);

            return await Task.FromResult(true);
        }

        public async Task<Item> GetItemAsync(string id)
        {
            return await Task.FromResult(items.FirstOrDefault(s => s.Id == id));
        }

        public async Task<IEnumerable<Item>> GetItemsAsync(bool forceRefresh = false)
        {
            return await Task.FromResult(items);
        }
    }
}

redpajama_set_name: RedPajamaGithub


text:
package ucar.nc2.ncml;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.net.URL;
import java.util.ArrayList;
import java.util.Formatter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;
import org.jdom2.Element;
import org.jdom2.JDOMException;
import org.jdom2.Namespace;
import org.jdom2.input.SAXBuilder;
import org.jdom2.output.XMLOutputter;
import ucar.ma2.Array;
import ucar.ma2.DataType;
import ucar.ma2.InvalidRangeException;
import ucar.ma2.MAMath;
import ucar.ma2.Section;
import ucar.nc2.Attribute;
import ucar.nc2.Dimension;
import ucar.nc2.EnumTypedef;
import ucar.nc2.FileWriter2;
import ucar.nc2.Group;
import ucar.nc2.NetcdfFile;
import ucar.nc2.NetcdfFileSubclass;
import ucar.nc2.NetcdfFileWriter;
import ucar.nc2.Sequence;
import ucar.nc2.Structure;
import ucar.nc2.Variable;
import ucar.nc2.constants.CDM;
import ucar.nc2.dataset.*;
import ucar.nc2.util.AliasTranslator;
import ucar.nc2.util.CancelTask;
import ucar.nc2.util.IO;
import ucar.nc2.util.URLnaming;
import ucar.nc2.write.Nc4Chunking;
import ucar.unidata.util.StringUtil2;
import static ucar.unidata.util.StringUtil2.getTokens;

/**
 * Read NcML and create NetcdfDataset.
 *
 * @author caron
 * @see https://www.unidata.ucar.edu/software/netcdf/ncml/
 * @deprecated read NcML using NetcdfDatasets.open()
 */
@Deprecated
public class NcMLReader {
  private static final Namespace ncNSHttp = thredds.client.catalog.Catalog.ncmlNS;
  private static final Namespace ncNSHttps = thredds.client.catalog.Catalog.ncmlNSHttps;
  private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(NcMLReader.class);

  private Namespace ncNS;

  private static boolean debugURL, debugXML, showParsedXML;
  private static boolean debugOpen, debugConstruct, debugCmd;
  private static boolean debugAggDetail;

  public static void setDebugFlags(ucar.nc2.util.DebugFlags debugFlag) {
    debugURL = debugFlag.isSet("NcML/debugURL");
    debugXML = debugFlag.isSet("NcML/debugXML");
    showParsedXML = debugFlag.isSet("NcML/showParsedXML");
    debugCmd = debugFlag.isSet("NcML/debugCmd");
    debugOpen = debugFlag.isSet("NcML/debugOpen");
    debugConstruct = debugFlag.isSet("NcML/debugConstruct");
    debugAggDetail = debugFlag.isSet("NcML/debugAggDetail");
  }

  /**
   * Use NCML to modify a dataset, getting the NcML document as a resource stream.
   * Uses ClassLoader.getResourceAsStream(ncmlResourceLocation), so the NcML can be inside of a jar file, for example.
   *
   * @param ncDataset modify this dataset
   * @param ncmlResourceLocation resource location of NcML
   * @param cancelTask allow user to cancel task; may be null
   * @throws IOException on read error
   */
  public static void wrapNcMLresource(NetcdfDataset ncDataset, String ncmlResourceLocation, CancelTask cancelTask)
      throws IOException {
    ClassLoader cl = ncDataset.getClass().getClassLoader();
    try (InputStream is = cl.getResourceAsStream(ncmlResourceLocation)) {
      if (is == null)
        throw new FileNotFoundException(ncmlResourceLocation);

      if (debugXML) {
        System.out.println(" NetcdfDataset URL = <" + ncmlResourceLocation + ">");
        try (InputStream is2 = cl.getResourceAsStream(ncmlResourceLocation)) {
          System.out.println(" contents=\n" + IO.readContents(is2));
        }
      }

      org.jdom2.Document doc;
      try {
        SAXBuilder builder = new SAXBuilder();
        builder.setExpandEntities(false);
        if (debugURL)
          System.out.println(" NetcdfDataset URL = <" + ncmlResourceLocation + ">");
        doc = builder.build(is);
      } catch (JDOMException e) {
        throw new IOException(e.getMessage());
      }
      if (debugXML)
        System.out.println(" SAXBuilder done");

      if (showParsedXML) {
        XMLOutputter xmlOut = new XMLOutputter();
        System.out.println("*** NetcdfDataset/showParsedXML = \n" + xmlOut.outputString(doc) + "\n*******");
      }

      Element netcdfElem = doc.getRootElement();

      NcMLReader reader = new NcMLReader();
      reader.readNetcdf(ncDataset.getLocation(), ncDataset, ncDataset, netcdfElem, cancelTask);
      if (debugOpen)
        System.out.println("***NcMLReader.wrapNcML result= \n" + ncDataset);
    }
  }

  /**
   * Use NCML to modify the dataset, getting NcML from a URL
   *
   * @param ncDataset modify this dataset
   * @param ncmlLocation URL location of NcML
   * @param cancelTask allow user to cancel task; may be null
   * @throws IOException on read error
   */
  public static void wrapNcML(NetcdfDataset ncDataset, String ncmlLocation, CancelTask cancelTask) throws IOException {
    org.jdom2.Document doc;
    try {
      SAXBuilder builder = new SAXBuilder();
      builder.setExpandEntities(false);
      if (debugURL)
        System.out.println(" NetcdfDataset URL = <" + ncmlLocation + ">");
      doc = builder.build(ncmlLocation);
    } catch (JDOMException e) {
      throw new IOException(e.getMessage());
    }
    if (debugXML)
      System.out.println(" SAXBuilder done");

    if (showParsedXML) {
      XMLOutputter xmlOut = new XMLOutputter();
      System.out.println("*** NetcdfDataset/showParsedXML = \n" + xmlOut.outputString(doc) + "\n*******");
    }

    Element netcdfElem = doc.getRootElement();

    NcMLReader reader = new NcMLReader();
    reader.readNetcdf(ncmlLocation, ncDataset, ncDataset, netcdfElem, cancelTask);
    if (debugOpen)
      System.out.println("***NcMLReader.wrapNcML result= \n" + ncDataset);
  }

  /**
   * Use NCML to modify the referenced dataset, create a new dataset with the merged info
   * Used to wrap each dataset of an aggregation before its aggregated
   *
   * @param ref referenced dataset
   * @param parentElem parent element - usually the aggregation element of the ncml
   * @return new dataset with the merged info
   * @throws IOException on read error
   */
  public static NetcdfDataset mergeNcML(NetcdfFile ref, Element parentElem) throws IOException {
    NetcdfDataset targetDS = new NetcdfDataset(ref, null); // no enhance

    NcMLReader reader = new NcMLReader();
    reader.readGroup(targetDS, targetDS, null, null, parentElem);
    targetDS.finish();

    return targetDS;
  }

  /**
   * Use NCML to directly modify the dataset
   *
   * @param targetDS referenced dataset
   * @param parentElem parent element - usually the aggregation element of the ncml
   * @return new dataset with the merged info
   */
  public static NetcdfDataset mergeNcMLdirect(NetcdfDataset targetDS, Element parentElem) {

    NcMLReader reader = new NcMLReader();
    reader.readGroup(targetDS, targetDS, null, null, parentElem);
    targetDS.finish();

    return targetDS;
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////////////

  /**
   * Read an NcML file from a URL location, and construct a NetcdfDataset.
   *
   * @param ncmlLocation the URL location string of the NcML document
   * @param cancelTask allow user to cancel the task; may be null
   * @return the resulting NetcdfDataset
   * @throws IOException on read error, or bad referencedDatasetUri URI
   */
  public static NetcdfDataset readNcML(String ncmlLocation, CancelTask cancelTask) throws IOException {
    return readNcML(ncmlLocation, (String) null, cancelTask);
  }

  /**
   * Read an NcML file from a URL location, and construct a NetcdfDataset.
   *
   * @param ncmlLocation the URL location string of the NcML document
   * @param referencedDatasetUri if null (usual case) get this from NcML, otherwise use URI as the location of the
   *        referenced dataset.
   * @param cancelTask allow user to cancel the task; may be null
   * @return the resulting NetcdfDataset
   * @throws IOException on read error, or bad referencedDatasetUri URI
   */
  public static NetcdfDataset readNcML(String ncmlLocation, String referencedDatasetUri, CancelTask cancelTask)
      throws IOException {
    URL url = new URL(ncmlLocation);

    if (debugURL) {
      System.out.println(" NcMLReader open " + ncmlLocation);
      System.out.println("   URL = " + url);
      System.out.println("   external form = " + url.toExternalForm());
      System.out.println("   protocol = " + url.getProtocol());
      System.out.println("   host = " + url.getHost());
      System.out.println("   path = " + url.getPath());
      System.out.println("  file = " + url.getFile());
    }

    org.jdom2.Document doc;
    try {
      SAXBuilder builder = new SAXBuilder();
      builder.setExpandEntities(false);
      if (debugURL)
        System.out.println(" NetcdfDataset URL = <" + url + ">");
      doc = builder.build(url);
    } catch (JDOMException e) {
      throw new IOException(e.getMessage());
    }
    if (debugXML)
      System.out.println(" SAXBuilder done");

    if (showParsedXML) {
      XMLOutputter xmlOut = new XMLOutputter();
      System.out.println("*** NetcdfDataset/showParsedXML = \n" + xmlOut.outputString(doc) + "\n*******");
    }

    Element netcdfElem = doc.getRootElement();

    if (referencedDatasetUri == null) {
      // the ncml probably refers to another dataset, but doesnt have to
      referencedDatasetUri = netcdfElem.getAttributeValue("location");
      if (referencedDatasetUri == null)
        referencedDatasetUri = netcdfElem.getAttributeValue("url");
    }
    if (referencedDatasetUri != null)
      referencedDatasetUri = AliasTranslator.translateAlias(referencedDatasetUri);

    NcMLReader reader = new NcMLReader();
    NetcdfDataset ncd = reader._readNcML(ncmlLocation, referencedDatasetUri, netcdfElem, cancelTask);
    if (debugOpen)
      System.out.println("***NcMLReader.readNcML result= \n" + ncd);
    return ncd;
  }

  /**
   * Read NcML doc from an InputStream, and construct a NetcdfDataset.
   *
   * @param ins the InputStream containing the NcML document
   * @param cancelTask allow user to cancel the task; may be null
   * @return the resulting NetcdfDataset
   * @throws IOException on read error, or bad referencedDatasetUri URI
   */
  public static NetcdfDataset readNcML(InputStream ins, CancelTask cancelTask) throws IOException {

    org.jdom2.Document doc;
    try {
      SAXBuilder builder = new SAXBuilder();
      builder.setExpandEntities(false);
      doc = builder.build(ins);
    } catch (JDOMException e) {
      throw new IOException(e.getMessage());
    }
    if (debugXML)
      System.out.println(" SAXBuilder done");

    if (showParsedXML) {
      XMLOutputter xmlOut = new XMLOutputter();
      System.out.println("*** NetcdfDataset/showParsedXML = \n" + xmlOut.outputString(doc) + "\n*******");
    }

    Element netcdfElem = doc.getRootElement();
    NetcdfDataset ncd = readNcML(null, netcdfElem, cancelTask);
    if (debugOpen)
      System.out.println("***NcMLReader.readNcML (stream) result= \n" + ncd);
    return ncd;
  }

  /**
   * Read NcML doc from a Reader, and construct a NetcdfDataset.
   *
   * @param r the Reader containing the NcML document
   * @param cancelTask allow user to cancel the task; may be null
   * @return the resulting NetcdfDataset
   * @throws IOException on read error, or bad referencedDatasetUri URI
   */
  public static NetcdfDataset readNcML(Reader r, CancelTask cancelTask) throws IOException {
    return readNcML(r, "NcMLReader", cancelTask);
  }

  /**
   * Read NcML doc from a Reader, and construct a NetcdfDataset.
   * eg: NcMLReader.readNcML(new StringReader(ncml), location, null);
   *
   * @param r the Reader containing the NcML document
   * @param ncmlLocation the URL location string of the NcML document, used to resolve relative path of the referenced
   *        dataset,
   *        or may be just a unique name for caching purposes.
   * @param cancelTask allow user to cancel the task; may be null
   * @return the resulting NetcdfDataset
   * @throws IOException on read error, or bad referencedDatasetUri URI
   */
  public static NetcdfDataset readNcML(Reader r, String ncmlLocation, CancelTask cancelTask) throws IOException {

    org.jdom2.Document doc;
    try {
      SAXBuilder builder = new SAXBuilder();
      builder.setExpandEntities(false);
      doc = builder.build(r);
    } catch (JDOMException e) {
      throw new IOException(e.getMessage());
    }
    if (debugXML)
      System.out.println(" SAXBuilder done");

    if (showParsedXML) {
      XMLOutputter xmlOut = new XMLOutputter();
      System.out.println("*** NetcdfDataset/showParsedXML = \n" + xmlOut.outputString(doc) + "\n*******");
    }

    Element netcdfElem = doc.getRootElement();
    NetcdfDataset ncd = readNcML(ncmlLocation, netcdfElem, cancelTask);
    if (debugOpen)
      System.out.println("***NcMLReader.readNcML (stream) result= \n" + ncd);
    return ncd;
  }

  /**
   * Read NcML from a JDOM Document, and construct a NetcdfDataset.
   *
   * @param ncmlLocation the URL location string of the NcML document, used to resolve relative path of the referenced
   *        dataset,
   *        or may be just a unique name for caching purposes.
   * @param netcdfElem the JDOM Document's root (netcdf) element
   * @param cancelTask allow user to cancel the task; may be null
   * @return the resulting NetcdfDataset
   * @throws IOException on read error, or bad referencedDatasetUri URI
   */
  public static NetcdfDataset readNcML(String ncmlLocation, Element netcdfElem, CancelTask cancelTask)
      throws IOException {
    // the ncml probably refers to another dataset, but doesnt have to
    String referencedDatasetUri = netcdfElem.getAttributeValue("location");
    if (referencedDatasetUri == null)
      referencedDatasetUri = netcdfElem.getAttributeValue("url");
    if (referencedDatasetUri != null)
      referencedDatasetUri = AliasTranslator.translateAlias(referencedDatasetUri);

    NcMLReader reader = new NcMLReader();
    return reader._readNcML(ncmlLocation, referencedDatasetUri, netcdfElem, cancelTask);
  }

  /**
   * Read NcML from a JDOM Document, and pass in the name of the dataset. Used to augment datasetScan with NcML
   *
   * @param ncmlLocation the URL location string of the NcML document, used as a unique name for caching purposes.
   * @param netcdfElem the JDOM Document's root (netcdf) element
   * @param referencedDatasetUri the URL location string of the underlying dataset, which overrides anything in
   *        netcdfElem.
   *        prepend with "file:" to eliminate relative resolving against ncmlLocation
   * @param cancelTask allow user to cancel the task; may be null
   * @return the resulting NetcdfDataset
   * @throws IOException on read error, or bad referencedDatasetUri URI
   */
  public static NetcdfDataset readNcML(String ncmlLocation, Element netcdfElem, String referencedDatasetUri,
      CancelTask cancelTask) throws IOException {
    NcMLReader reader = new NcMLReader();
    return reader._readNcML(ncmlLocation, referencedDatasetUri, netcdfElem, cancelTask);
  }

  //////////////////////////////////////////////////////////////////////////////////////
  private String location;
  private boolean explicit;
  private Formatter errlog = new Formatter();

  /**
   * This sets up the target dataset and the referenced dataset.
   * only place that iospParam is processed, so everything must go through here
   *
   * @param ncmlLocation the URL location string of the NcML document, used to resolve relative path of the referenced
   *        dataset,
   *        or may be just a unique name for caching purposes.
   * @param referencedDatasetUri refers to this dataset (may be null)
   * @param netcdfElem JDOM netcdf element
   * @param cancelTask allow user to cancel the task; may be null
   * @return NetcdfDataset the constructed dataset
   * @throws IOException on read error, or bad referencedDatasetUri URI
   */
  private NetcdfDataset _readNcML(String ncmlLocation, String referencedDatasetUri, Element netcdfElem,
      CancelTask cancelTask) throws IOException {

    // get ncml namespace and set namespace variable
    this.ncNS = ncNSHttp;
    if (netcdfElem.getNamespaceURI().startsWith("https")) {
      this.ncNS = ncNSHttps;
    }

    // augment URI.resolve(), by also dealing with base file: URIs
    referencedDatasetUri = URLnaming.resolve(ncmlLocation, referencedDatasetUri);

    // common error causing infinite regression
    if ((referencedDatasetUri != null) && referencedDatasetUri.equals(ncmlLocation))
      throw new IllegalArgumentException(
          "NcML location attribute refers to the NcML document itself" + referencedDatasetUri);

    // they can specify the iosp to use - but must be file based
    String iospS = netcdfElem.getAttributeValue("iosp");
    Object iospParam = netcdfElem.getAttributeValue("iospParam");
    if (iospParam == null) {
      // can pass iosp a JDOM tree
      iospParam = netcdfElem.getChild("iospParam", ncNS); // LOOK namespace ??
    }

    String bufferSizeS = netcdfElem.getAttributeValue("buffer_size");
    int buffer_size = -1;
    if (bufferSizeS != null)
      buffer_size = Integer.parseInt(bufferSizeS);

    // open the referenced dataset - do NOT use acquire, and dont enhance
    // LOOK : shouldnt enhance be controlled by enhance attribute on the netcdf element ?
    NetcdfDataset refds = null;
    if (referencedDatasetUri != null) {
      if (iospS != null) {
        NetcdfFile ncfile;
        try {
          ncfile = new NetcdfFileSubclass(iospS, iospParam, referencedDatasetUri, buffer_size, cancelTask);
        } catch (Exception e) {
          throw new IOException(e);
        }
        refds = new NetcdfDataset(ncfile, false);
      } else {
        refds = NetcdfDataset.openDataset(referencedDatasetUri, false, buffer_size, cancelTask, iospParam);
        // refds.setEnhanceProcessed(false); // hasnt had enhance applied to it yet - wait till ncml mods have been
        // applied
      }
    }

    // explicit means all of the metadata is specified in the XML, and the referenced dataset is used only for data
    // access
    Element elemE = netcdfElem.getChild("explicit", ncNS);
    explicit = (elemE != null);

    // general idea is that we just modify the referenced dataset
    // the exception is when explicit is specified, then we keep them separate.
    // refds != null refds == null
    // explicit refds!=new new (ref=new)
    // readMetadata modify (new=ref) new (ref=new)
    //
    NetcdfDataset targetDS;
    if (explicit || (refds == null)) {
      targetDS = new NetcdfDataset();
      if (refds == null)
        refds = targetDS;
      else
        targetDS.setReferencedFile(refds); // gotta set so it gets closed !!

    } else { // modify the referenced dataset directly
      targetDS = refds;
    }

    // continue processing here
    readNetcdf(ncmlLocation, targetDS, refds, netcdfElem, cancelTask);

    return targetDS;
  }

  ///////// Heres where the parsing work starts

  /**
   * parse a netcdf JDOM Element, and add contents to the targetDS NetcdfDataset.
   *

   * This is a bit tricky, because it handles several cases
   * When targetDS == refds, we are just modifying targetDS.
   * When targetDS != refds, we keep them separate, and copy from refds to newds.
   *

   * The user may be defining new elements or modifying old ones. The only way to tell is by seeing
   * if the elements already exist.
   *
   * @param ncmlLocation NcML URL location, or may be just a unique name for caching purposes.
   * @param targetDS add the info to this one, never null
   * @param refds the referenced dataset; may equal targetDS, never null
   * @param netcdfElem JDOM netcdf element
   * @param cancelTask allow user to cancel the task; may be null
   * @throws IOException on read error
   */
  private void readNetcdf(String ncmlLocation, NetcdfDataset targetDS, NetcdfFile refds, Element netcdfElem,
      CancelTask cancelTask) throws IOException {
    this.location = ncmlLocation; // log messages need this

    if (debugOpen)
      System.out
          .println("NcMLReader.readNetcdf ncml= " + ncmlLocation + " referencedDatasetUri= " + refds.getLocation());

    // detect incorrect namespace
    Namespace use = netcdfElem.getNamespace();
    if (!use.equals(ncNSHttp) && !use.equals(ncNSHttps)) {
      String message = String.format("Namespace specified in NcML must be either '%s' or '%s', but was '%s'.",
          ncNSHttp.getURI(), ncNSHttps.getURI(), use.getURI());
      throw new IllegalArgumentException(message);
    }

    if (ncmlLocation != null)
      targetDS.setLocation(ncmlLocation);
    targetDS.setId(netcdfElem.getAttributeValue("id"));
    targetDS.setTitle(netcdfElem.getAttributeValue("title"));

    // aggregation first
    Element aggElem = netcdfElem.getChild("aggregation", ncNS);
    if (aggElem != null) {
      Aggregation agg = readAgg(aggElem, ncmlLocation, targetDS, cancelTask);
      if (agg == null)
        return; // cancel task
      targetDS.setAggregation(agg);
      agg.finish(cancelTask);
    }

    // the root group
    readGroup(targetDS, refds, null, null, netcdfElem);
    String errors = errlog.toString();
    if (!errors.isEmpty())
      throw new IllegalArgumentException("NcML had fatal errors:" + errors);

    // transfer from groups to global containers
    targetDS.finish();

    // enhance means do scale/offset and/or add CoordSystems
    Set<NetcdfDataset.Enhance> mode = NetcdfDataset.parseEnhanceMode(netcdfElem.getAttributeValue("enhance"));
    // if (mode == null)
    // mode = NetcdfDataset.getEnhanceDefault();
    targetDS.enhance(mode);

    // optionally add record structure to netcdf-3
    String addRecords = netcdfElem.getAttributeValue("addRecords");
    if ("true".equalsIgnoreCase(addRecords))
      targetDS.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
  }

  ////////////////////////////////////////////////////////////////////////

  /**
   * Read an NcML attribute element.
   *
   * @param parent Group or Variable
   * @param refParent Group or Variable in reference dataset
   * @param attElem ncml attribute element
   */
  private void readAtt(Object parent, Object refParent, Element attElem) {
    String name = attElem.getAttributeValue("name");
    if (name == null) {
      errlog.format("NcML Attribute name is required (%s)%n", attElem);
      return;
    }
    String nameInFile = attElem.getAttributeValue("orgName");
    boolean newName = (nameInFile != null) && !nameInFile.equals(name);
    if (nameInFile == null)
      nameInFile = name;
    else if (null == findAttribute(refParent, nameInFile)) { // has to exists
      errlog.format("NcML attribute orgName '%s' doesnt exist. att=%s in=%s%n", nameInFile, name, parent);
      return;
    }

    // see if its new
    ucar.nc2.Attribute oldatt = findAttribute(refParent, nameInFile);
    if (oldatt == null) { // new
      if (debugConstruct)
        System.out.println(" add new att = " + name);
      try {
        ucar.ma2.Array values = readAttributeValues(attElem);
        addAttribute(parent, new ucar.nc2.Attribute(name, values));
      } catch (RuntimeException e) {
        errlog.format("NcML new Attribute Exception: %s att=%s in=%s%n", e.getMessage(), name, parent);
      }

    } else { // already exists

      if (debugConstruct)
        System.out.println(" modify existing att = " + name);
      boolean hasValue = attElem.getAttribute("value") != null;
      if (hasValue) { // has a new value
        try {
          ucar.ma2.Array values = readAttributeValues(attElem); // Handles "isUnsigned".
          addAttribute(parent, new ucar.nc2.Attribute(name, values));
        } catch (RuntimeException e) {
          errlog.format("NcML existing Attribute Exception: %s att=%s in=%s%n", e.getMessage(), name, parent);
          return;
        }

      } else { // use the old values
        Array oldval = oldatt.getValues();
        if (oldval != null)
          addAttribute(parent, new ucar.nc2.Attribute(name, oldatt.getValues()));
        else { // weird corner case of attribute with no value - must use the type
          String unS = attElem.getAttributeValue("isUnsigned"); // deprecated but must deal with
          boolean isUnsignedSet = "true".equalsIgnoreCase(unS);
          String typeS = attElem.getAttributeValue("type");
          DataType dtype = typeS == null ? DataType.STRING : DataType.getType(typeS);
          if (isUnsignedSet)
            dtype = dtype.withSignedness(DataType.Signedness.UNSIGNED);
          addAttribute(parent, new ucar.nc2.Attribute(name, dtype));
        }
      }

      // remove the old one ??
      if (newName && !explicit) {
        removeAttribute(parent, oldatt);
        if (debugConstruct)
          System.out.println(" remove old att = " + nameInFile);
      }

    }
  }

  /**
   * Parse the values element
   *
   * @param s JDOM element to parse
   * @return Array with parsed values
   * @throws IllegalArgumentException if string values not parsable to specified data type
   */
  public static ucar.ma2.Array readAttributeValues(Element s) throws IllegalArgumentException {
    String valString = s.getAttributeValue("value");

    // can also be element text
    if (valString == null) {
      valString = s.getTextNormalize();
    }

    // no value specified hmm technically this is not illegal !!
    if (valString == null)
      throw new IllegalArgumentException("No value specified");

    String type = s.getAttributeValue("type");
    DataType dtype = (type == null) ? DataType.STRING : DataType.getType(type);
    if (dtype == DataType.CHAR)
      dtype = DataType.STRING;

    // backwards compatibility with deprecated isUnsigned attribute
    String unS = s.getAttributeValue("isUnsigned");
    boolean isUnsignedSet = "true".equalsIgnoreCase(unS);
    if (isUnsignedSet && dtype.isIntegral() && !dtype.isUnsigned()) {
      dtype = dtype.withSignedness(DataType.Signedness.UNSIGNED);
    }

    String sep = s.getAttributeValue("separator");
    if ((sep == null) && (dtype == DataType.STRING)) {
      List<String> list = new ArrayList<>();
      list.add(valString);
      return Array.makeArray(dtype, list);
    }

    if (sep == null)
      sep = " "; // default whitespace separated

    List<String> stringValues = new ArrayList<>();
    StringTokenizer tokn = new StringTokenizer(valString, sep);
    while (tokn.hasMoreTokens())
      stringValues.add(tokn.nextToken());

    return Array.makeArray(dtype, stringValues);
  }

  private ucar.nc2.Attribute findAttribute(Object parent, String name) {
    if (parent == null)
      return null;
    if (parent instanceof Group)
      return ((Group) parent).findAttribute(name);
    else if (parent instanceof Variable)
      return ((Variable) parent).findAttribute(name);
    return null;
  }

  private void addAttribute(Object parent, ucar.nc2.Attribute att) {
    if (parent instanceof Group)
      ((Group) parent).addAttribute(att);
    else if (parent instanceof Variable)
      ((Variable) parent).addAttribute(att);
  }

  private void removeAttribute(Object parent, Attribute att) {
    if (parent instanceof Group)
      ((Group) parent).remove(att);
    else if (parent instanceof Variable)
      ((Variable) parent).remove(att);
  }

  /**
   * Read an NcML dimension element.
   *
   * @param g put dimension into this group
   * @param refg parent Group in referenced dataset
   * @param dimElem ncml dimension element
   */
  private void readDim(Group g, Group refg, Element dimElem) {
    String name = dimElem.getAttributeValue("name");
    if (name == null) {
      errlog.format("NcML Dimension name is required (%s)%n", dimElem);
      return;
    }

    String nameInFile = dimElem.getAttributeValue("orgName");
    if (nameInFile == null)
      nameInFile = name;

    // see if it already exists
    Dimension dim = (refg == null) ? null : refg.findDimension(nameInFile);
    if (dim == null) { // nope - create it
      String lengthS = dimElem.getAttributeValue("length");
      if (lengthS == null) {
        errlog.format("NcML Dimension length is required (%s)%n", dimElem);
        return;
      }

      String isUnlimitedS = dimElem.getAttributeValue("isUnlimited");
      String isSharedS = dimElem.getAttributeValue("isShared");
      String isUnknownS = dimElem.getAttributeValue("isVariableLength");

      boolean isUnlimited = "true".equalsIgnoreCase(isUnlimitedS);
      boolean isUnknown = "true".equalsIgnoreCase(isUnknownS);
      boolean isShared = true;
      if ("false".equalsIgnoreCase(isSharedS))
        isShared = false;

      int len;
      if (isUnknown) {
        len = Dimension.VLEN.getLength();
      } else {
        len = Integer.parseInt(lengthS);
      }

      if (debugConstruct)
        System.out.println(" add new dim = " + name);
      g.addDimension(new Dimension(name, len, isShared, isUnlimited, isUnknown));

    } else { // yes - modify it
      dim.setName(name);

      String lengthS = dimElem.getAttributeValue("length");
      String isUnlimitedS = dimElem.getAttributeValue("isUnlimited");
      String isSharedS = dimElem.getAttributeValue("isShared");
      String isUnknownS = dimElem.getAttributeValue("isVariableLength");

      if (isUnlimitedS != null)
        dim.setUnlimited(isUnlimitedS.equalsIgnoreCase("true"));

      if (isSharedS != null)
        dim.setShared(!isSharedS.equalsIgnoreCase("false"));

      if (isUnknownS != null)
        dim.setVariableLength(isUnknownS.equalsIgnoreCase("true"));

      if ((lengthS != null) && !dim.isVariableLength()) {
        int len = Integer.parseInt(lengthS);
        dim.setLength(len);
      }

      if (debugConstruct)
        System.out.println(" modify existing dim = " + name);

      if (g != refg) // explicit, copy to new
        g.addDimension(dim);
    }
  }

  /**
   * Read an NcML enumTypedef element.
   *
   * @param g put enumTypedef into this group
   * @param refg parent Group in referenced dataset
   * @param etdElem ncml enumTypedef element
   */
  private void readEnumTypedef(Group g, Group refg, Element etdElem) {
    String name = etdElem.getAttributeValue("name");
    if (name == null) {
      errlog.format("NcML enumTypedef name is required (%s)%n", etdElem);
      return;
    }
    String typeS = etdElem.getAttributeValue("type");
    DataType baseType = (typeS == null) ? DataType.ENUM1 : DataType.getType(typeS);

    Map<Integer, String> map = new HashMap<>(100);
    for (Element e : etdElem.getChildren("enum", ncNS)) {
      String key = e.getAttributeValue("key");
      String value = e.getTextNormalize();
      if (key == null) {
        errlog.format("NcML enumTypedef enum key attribute is required (%s)%n", e);
        continue;
      }
      if (value == null) {
        errlog.format("NcML enumTypedef enum value is required (%s)%n", e);
        continue;
      }
      try {
        int keyi = Integer.parseInt(key);
        map.put(keyi, value);
      } catch (Exception e2) {
        errlog.format("NcML enumTypedef enum key attribute not an integer (%s)%n", e);
      }
    }

    EnumTypedef td = new EnumTypedef(name, map, baseType);
    g.addEnumeration(td);
  }

  /**
   * Read the NcML group element, and nested elements.
   *
   * @param newds new dataset
   * @param refds referenced dataset
   * @param parent Group
   * @param refParent parent Group in referenced dataset
   * @param groupElem ncml group element
   */
  private void readGroup(NetcdfDataset newds, NetcdfFile refds, Group parent, Group refParent, Element groupElem) {

    Group g, refg = null;
    if (parent == null) { // this is the <netcdf> element
      g = newds.getRootGroup();
      refg = refds.getRootGroup();
      if (debugConstruct)
        System.out.println(" root group ");

    } else {

      String name = groupElem.getAttributeValue("name");
      if (name == null) {
        errlog.format("NcML Group name is required (%s)%n", groupElem);
        return;
      }

      String nameInFile = groupElem.getAttributeValue("orgName");
      if (nameInFile == null)
        nameInFile = name;

      // see if it exists in referenced dataset
      if (refParent != null)
        refg = refParent.findGroupLocal(nameInFile);
      if (refg == null) { // new
        g = new Group(newds, parent, name);
        parent.addGroup(g);
        if (debugConstruct)
          System.out.println(" add new group = " + name);

      } else {

        if (parent != refParent) { // explicit
          g = new Group(newds, parent, name);
          parent.addGroup(g);
          if (debugConstruct)
            System.out.println(" transfer existing group = " + name);

        } else { // modify
          g = refg;
          if (!nameInFile.equals(name))
            g.setName(name);

          if (debugConstruct)
            System.out.println(" modify existing group = " + name);
        }
      }
    }

    // look for attributes
    java.util.List<Element> attList = groupElem.getChildren("attribute", ncNS);
    for (Element attElem : attList) {
      readAtt(g, refg, attElem);
    }

    // look for enumTypedef
    java.util.List<Element> etdList = groupElem.getChildren("enumTypedef", ncNS);
    for (Element elem : etdList) {
      readEnumTypedef(g, refg, elem);
    }

    // look for dimensions
    java.util.List<Element> dimList = groupElem.getChildren("dimension", ncNS);
    for (Element dimElem : dimList) {
      readDim(g, refg, dimElem);
    }

    // look for variables
    java.util.List<Element> varList = groupElem.getChildren("variable", ncNS);
    for (Element varElem : varList) {
      readVariable(newds, g, refg, varElem);
    }

    // process remove command
    java.util.List<Element> removeList = groupElem.getChildren("remove", ncNS);
    for (Element e : removeList) {
      cmdRemove(g, e.getAttributeValue("type"), e.getAttributeValue("name"));
    }

    // look for nested groups
    java.util.List<Element> groupList = groupElem.getChildren("group", ncNS);
    for (Element gElem : groupList) {
      readGroup(newds, refds, g, refg, gElem);
      if (debugConstruct)
        System.out.println(" add group = " + g.getFullName());
    }
  }

  /*
   * private boolean debugView = false, debugConvert = false;
   * protected VariableDS readVariable2( NetcdfDataset ds, Element varElem) {
   * VariableDS v = readVariable( ds, varElem);
   *
   * // look for logical views
   * java.util.List viewList = varElem.getChildren("logicalView", ncNS);
   * for (int j=0; j< viewList.size(); j++) {
   * Element viewElem = (Element) viewList.get(j);
   * String value = viewElem.getAttributeValue("section");
   * if (value != null) {
   * v.setLogicalView("section", value);
   * if (debugView) System.out.println("set view = "+value);
   * }
   * }
   *
   * // look for unit conversion
   * Element unitElem = varElem.getChild(CDM.UNITS, ncNS);
   * if (unitElem != null) {
   * String value = unitElem.getAttributeValue("convertTo");
   * if (value != null) {
   * v.setConvertUnit(value);
   * if (debugConvert) System.out.println("setConvertUnit on "+v.getName()+" to <" + value+">");
   * }
   * }
   *
   * return v;
   * }
   */

  /**
   * Read the NcML variable element, and nested elements.
   *
   * @param ds target dataset
   * @param g parent Group
   * @param refg referenced dataset parent Group - may be same (modify) or different (explicit)
   * @param varElem ncml variable element
   */
  private void readVariable(NetcdfDataset ds, Group g, Group refg, Element varElem) {
    String name = varElem.getAttributeValue("name");
    if (name == null) {
      errlog.format("NcML Variable name is required (%s)%n", varElem);
      return;
    }

    String nameInFile = varElem.getAttributeValue("orgName");
    if (nameInFile == null)
      nameInFile = name;

    // see if it already exists
    Variable refv = (refg == null) ? null : refg.findVariableLocal(nameInFile);
    if (refv == null) { // new
      if (debugConstruct)
        System.out.println(" add new var = " + name);
      g.addVariable(readVariableNew(ds, g, null, varElem));
      return;
    }

    // exists already
    DataType dtype;
    String typeS = varElem.getAttributeValue("type");
    if (typeS != null)
      dtype = DataType.getType(typeS);
    else
      dtype = refv.getDataType();

    EnumTypedef typedef = null;
    if (dtype.isEnum()) {
      String typedefS = varElem.getAttributeValue("typedef");
      if (typedefS != null)
        typedef = g.findEnumeration(typedefS);
    }

    String shape = varElem.getAttributeValue("shape");

    Variable v;
    if (refg == g) { // modify
      v = refv;
      v.setName(name);
      /*
       * if (dtype != v.getDataType() && v.hasCachedData()) {
       * Array data = v.read();
       * Array newData = Array.factory(dtype, v.getShape());
       * MAMath.copy(newData, data);
       * v.setCachedData(newData, false);
       * }
       */
      v.setDataType(dtype);
      if (typedef != null)
        v.setEnumTypedef(typedef);

      if (shape != null)
        v.setDimensions(shape); // LOOK check conformable
      if (debugConstruct)
        System.out.println(" modify existing var = " + nameInFile);

    } else { // explicit - create new
      if (refv instanceof Structure) {
        v = new StructureDS(ds, g, null, name, (Structure) refv);
        v.setDimensions(shape);
      } else {
        v = new VariableDS(g, null, name, refv);
        v.setDataType(dtype);
        v.setDimensions(shape);
      }
      if (debugConstruct)
        System.out.println(" modify explicit var = " + nameInFile);
      g.addVariable(v);
    }

    java.util.List<Element> attList = varElem.getChildren("attribute", ncNS);
    for (Element attElem : attList) {
      readAtt(v, refv, attElem);
    }

    // deal with legacy use of attribute with Unsigned = true
    Attribute att = v.findAttribute(CDM.UNSIGNED);
    boolean isUnsignedSet = att != null && att.getStringValue().equalsIgnoreCase("true");
    if (isUnsignedSet) {
      dtype = dtype.withSignedness(DataType.Signedness.UNSIGNED);
      v.setDataType(dtype);
    }

    // process remove command
    java.util.List<Element> removeList = varElem.getChildren("remove", ncNS);
    for (Element remElem : removeList) {
      cmdRemove(v, remElem.getAttributeValue("type"), remElem.getAttributeValue("name"));
    }

    if (v.getDataType() == DataType.STRUCTURE) {
      // deal with nested variables
      StructureDS s = (StructureDS) v;
      StructureDS refS = (StructureDS) refv;
      java.util.List<Element> varList = varElem.getChildren("variable", ncNS);
      for (Element vElem : varList) {
        readVariableNested(ds, s, refS, vElem);
      }

    } else {

      // deal with values
      Element valueElem = varElem.getChild("values", ncNS);
      if (valueElem != null) {
        readValues(ds, v, varElem, valueElem);

      } else {
        // see if we need to munge existing data. use case : aggregation
        if (v.hasCachedData()) {
          Array data;
          try {
            data = v.read();
          } catch (IOException e) {
            throw new IllegalStateException(e.getMessage());
          }
          if (data.getClass() != v.getDataType().getPrimitiveClassType()) {
            Array newData = Array.factory(v.getDataType(), v.getShape());
            MAMath.copy(newData, data);
            v.setCachedData(newData, false);
          }
        }
      }
    }

    // look for logical views
    processLogicalViews(varElem, v, g);
  }

  private void processLogicalViews(Element varElem, Variable v, Group g) {

    Element viewElem = varElem.getChild("logicalSection", ncNS);
    if (null != viewElem) {
      String sectionSpec = viewElem.getAttributeValue("section");
      if (sectionSpec != null) {
        try {
          Section s = new Section(sectionSpec); // parse spec
          Section viewSection = Section.fill(s, v.getShape());
          // check that its a subset
          if (!v.getShapeAsSection().contains(viewSection)) {
            errlog.format("Invalid logicalSection on variable=%s section =(%s) original=(%s) %n", v.getFullName(),
                sectionSpec, v.getShapeAsSection());
            return;
          }
          Variable view = v.section(viewSection);
          g.removeVariable(v.getShortName());
          g.addVariable(view);

        } catch (InvalidRangeException e) {
          errlog.format("Invalid logicalSection on variable=%s section=(%s) error=%s %n", v.getFullName(), sectionSpec,
              e.getMessage());
          return;
        }
      }
    }

    viewElem = varElem.getChild("logicalSlice", ncNS);
    if (null != viewElem) {
      String dimName = viewElem.getAttributeValue("dimName");
      if (null == dimName) {
        errlog.format("NcML logicalSlice: dimName is required, variable=%s %n", v.getFullName());
        return;
      }
      int dim = v.findDimensionIndex(dimName);
      if (dim < 0) {
        errlog.format("NcML logicalSlice: cant find dimension %s in variable=%s %n", dimName, v.getFullName());
        return;
      }

      String indexS = viewElem.getAttributeValue("index");
      int index;
      if (null == indexS) {
        errlog.format("NcML logicalSlice: index is required, variable=%s %n", v.getFullName());
        return;
      }
      try {
        index = Integer.parseInt(indexS);
      } catch (NumberFormatException e) {
        errlog.format("NcML logicalSlice: index=%s must be integer, variable=%s %n", indexS, v.getFullName());
        return;
      }

      try {
        Variable view = v.slice(dim, index);
        g.removeVariable(v.getShortName());
        g.addVariable(view);

      } catch (InvalidRangeException e) {
        errlog.format("Invalid logicalSlice (%d,%d) on variable=%s error=%s %n", dim, index, v.getFullName(),
            e.getMessage());
      }
    }

    viewElem = varElem.getChild("logicalReduce", ncNS);
    if (null != viewElem) {
      String dimName = viewElem.getAttributeValue("dimNames");
      if (null == dimName) {
        errlog.format("NcML logicalReduce: dimNames is required, variable=%s %n", v.getFullName());
        return;
      }
      String[] dims = StringUtil2.splitString(dimName);
      List<Dimension> dimList = new ArrayList<>();
      for (String s : dims) {
        int idx = v.findDimensionIndex(s);
        if (idx < 0) {
          errlog.format("NcML logicalReduce: cant find dimension %s in variable=%s %n", dimName, v.getFullName());
          return;
        }
        dimList.add(v.getDimension(idx));
      }

      Variable view = v.reduce(dimList);
      g.removeVariable(v.getShortName());
      g.addVariable(view);
    }

  }

  /**
   * Read a NcML variable element, and nested elements, when it creates a new Variable.
   *
   * @param ds target dataset
   * @param g parent Group
   * @param parentS parent Structure
   * @param varElem ncml variable element
   * @return return new Variable
   */
  private Variable readVariableNew(NetcdfDataset ds, Group g, Structure parentS, Element varElem) {
    String name = varElem.getAttributeValue("name");
    if (name == null) {
      errlog.format("NcML Variable name is required (%s)%n", varElem);
      return null;
    }

    String type = varElem.getAttributeValue("type");
    if (type == null)
      throw new IllegalArgumentException("New variable (" + name + ") must have datatype attribute");
    DataType dtype = DataType.getType(type);

    EnumTypedef typedef = null;
    if (dtype.isEnum()) {
      String typedefS = varElem.getAttributeValue("typedef");
      if (typedefS != null)
        typedef = g.findEnumeration(typedefS);
    }

    String shape = varElem.getAttributeValue("shape");
    if (shape == null)
      shape = ""; // deprecated, prefer explicit ""

    Variable v;

    if (dtype == DataType.STRUCTURE) {
      StructureDS s = new StructureDS(ds, g, parentS, name, shape, null, null);
      v = s;
      // look for nested variables
      java.util.List<Element> varList = varElem.getChildren("variable", ncNS);
      for (Element vElem : varList) {
        readVariableNested(ds, s, s, vElem);
      }

    } else if (dtype == DataType.SEQUENCE) {
      Sequence org = new Sequence(ds, g, parentS, name);
      SequenceDS s = new SequenceDS(g, org); // barf
      v = s;
      // look for nested variables
      java.util.List<Element> varList = varElem.getChildren("variable", ncNS);
      for (Element vElem : varList) {
        readVariableNested(ds, s, s, vElem);
      }

    } else {
      v = new VariableDS(ds, g, parentS, name, dtype, shape, null, null);

      // deal with values
      Element valueElem = varElem.getChild("values", ncNS);
      if (valueElem != null)
        readValues(ds, v, varElem, valueElem);
      // otherwise has fill values.
    }

    // look for attributes
    java.util.List<Element> attList = varElem.getChildren("attribute", ncNS);
    for (Element attElem : attList)
      readAtt(v, null, attElem);

    if (typedef != null)
      v.setEnumTypedef(typedef);

    /*
     * now that we have attributes finalized, redo the enhance
     * LOOK enhance not set
     * if (enhance && (v instanceof VariableDS)) {
     * ((VariableDS) v).enhance();
     * }
     */

    return v;
  }

  /**
   * Read the NcML variable element, and nested elements.
   *
   * @param ds target dataset
   * @param parentS parent Structure
   * @param refStruct reference dataset structure
   * @param varElem ncml variable element
   */
  private void readVariableNested(NetcdfDataset ds, Structure parentS, Structure refStruct, Element varElem) {
    String name = varElem.getAttributeValue("name");
    if (name == null) {
      errlog.format("NcML Variable name is required (%s)%n", varElem);
      return;
    }

    String nameInFile = varElem.getAttributeValue("orgName");
    if (nameInFile == null)
      nameInFile = name;

    // see if it already exists
    Variable refv = refStruct.findVariable(nameInFile);
    if (refv == null) { // new
      if (debugConstruct)
        System.out.println(" add new var = " + name);
      Variable nested = readVariableNew(ds, parentS.getParentGroupOrRoot(), parentS, varElem);
      if (nested != null)
        parentS.addMemberVariable(nested);
      return;
    }

    Variable v;
    if (parentS == refStruct) { // modify
      v = refv;
      v.setName(name);

    } else { // explicit
      if (refv instanceof Structure) {
        v = new StructureDS(parentS.getParentGroupOrRoot(), (Structure) refv); // true
        v.setName(name);
        v.setParentStructure(parentS);
      } else {
        v = new VariableDS(parentS.getParentGroupOrRoot(), refv, false);
        v.setName(name);
        v.setParentStructure(parentS);
      }

      /*
       * if (refv instanceof Structure) {
       * v = new StructureDS(ds, parentS.getParentGroup(), parentS, name, refv.getDimensionsString(), null, null);
       * } else {
       * v = new VariableDS(ds, parentS.getParentGroup(), parentS, name, refv.getDataType(), refv.getDimensionsString(),
       * null, null);
       * }
       * v.setIOVar(refv);
       */
      parentS.addMemberVariable(v);
    }

    if (debugConstruct)
      System.out.println(" modify existing var = " + nameInFile);

    String typeS = varElem.getAttributeValue("type");
    if (typeS != null) {
      DataType dtype = DataType.getType(typeS);
      v.setDataType(dtype);
    }

    String shape = varElem.getAttributeValue("shape");
    if (shape != null) {
      v.setDimensions(shape);
    }

    java.util.List<Element> attList = varElem.getChildren("attribute", ncNS);
    for (Element attElem : attList) {
      readAtt(v, refv, attElem);
    }

    // process remove command
    java.util.List<Element> removeList = varElem.getChildren("remove", ncNS);
    for (Element remElem : removeList) {
      cmdRemove(v, remElem.getAttributeValue("type"), remElem.getAttributeValue("name"));
    }

    if ((v.getDataType() == DataType.STRUCTURE) || (v.getDataType() == DataType.SEQUENCE)) {
      // deal with nested variables
      StructureDS s = (StructureDS) v;
      StructureDS refS = (StructureDS) refv;
      java.util.List<Element> varList = varElem.getChildren("variable", ncNS);
      for (Element vElem : varList) {
        readVariableNested(ds, s, refS, vElem);
      }

    } else {

      // deal with values
      Element valueElem = varElem.getChild("values", ncNS);
      if (valueElem != null)
        readValues(ds, v, varElem, valueElem);
    }

    /*
     * now that we have attributes finalized, redo the enhance
     * if (enhance && (v instanceof VariableDS))
     * ((VariableDS) v).enhance();
     */
  }

  private void readValues(NetcdfDataset ds, Variable v, Element varElem, Element valuesElem) {
    try {

      // check if values are specified by attribute
      String fromAttribute = valuesElem.getAttributeValue("fromAttribute");
      if (fromAttribute != null) {
        Attribute att;
        int pos = fromAttribute.indexOf('@'); // varName@attName
        if (pos > 0) {
          String varName = fromAttribute.substring(0, pos);
          String attName = fromAttribute.substring(pos + 1);
          Variable vFrom = ds.getRootGroup().findVariableLocal(varName); // LOOK groups
          if (vFrom == null) {
            errlog.format("Cant find variable %s %n", fromAttribute);
            return;
          }
          att = vFrom.findAttribute(attName);

        } else { // attName or @attName
          String attName = (pos == 0) ? fromAttribute.substring(1) : fromAttribute;
          att = ds.getRootGroup().findAttribute(attName);
        }
        if (att == null) {
          errlog.format("Cant find attribute %s %n", fromAttribute);
          return;
        }
        Array data = att.getValues();
        v.setCachedData(data, true);
        return;
      }

      // check if values are specified by start / increment
      String startS = valuesElem.getAttributeValue("start");
      String incrS = valuesElem.getAttributeValue("increment");
      String nptsS = valuesElem.getAttributeValue("npts");
      int npts = (nptsS == null) ? (int) v.getSize() : Integer.parseInt(nptsS);

      // either start, increment are specified
      if ((startS != null) && (incrS != null)) {
        double start = Double.parseDouble(startS);
        double incr = Double.parseDouble(incrS);
        v.setValues(npts, start, incr);
        return;
      }

      // otherwise values are listed in text
      String values = varElem.getChildText("values", ncNS);
      String sep = valuesElem.getAttributeValue("separator");

      if (v.getDataType() == DataType.CHAR) {
        int nhave = values.length();
        int nwant = (int) v.getSize();
        char[] data = new char[nwant];
        int min = Math.min(nhave, nwant);
        for (int i = 0; i < min; i++) {
          data[i] = values.charAt(i);
        }
        Array dataArray = Array.factory(DataType.CHAR, v.getShape(), data);
        v.setCachedData(dataArray, true);

      } else {
        List<String> valList = getTokens(values, sep);
        v.setValues(valList);
      }

    } catch (Throwable t) {
      throw new RuntimeException("NCML Reading on " + v.getFullName(), t);
    }
  }

  /////////////////////////////////////////////////////////////////////////////////////////

  private Aggregation readAgg(Element aggElem, String ncmlLocation, NetcdfDataset newds, CancelTask cancelTask) {
    String dimName = aggElem.getAttributeValue("dimName");
    String type = aggElem.getAttributeValue("type");
    String recheck = aggElem.getAttributeValue("recheckEvery");

    Aggregation agg;
    if (type.equalsIgnoreCase("joinExisting")) {
      agg = new AggregationExisting(newds, dimName, recheck);

    } else if (type.equalsIgnoreCase("joinNew")) {
      agg = new AggregationNew(newds, dimName, recheck);

    } else if (type.equalsIgnoreCase("tiled")) {
      agg = new AggregationTiled(newds, dimName, recheck);

    } else if (type.equalsIgnoreCase("union")) {
      agg = new AggregationUnion(newds, dimName, recheck);

    } else if (type.equalsIgnoreCase("forecastModelRunCollection")
        || type.equalsIgnoreCase("forecastModelRunSingleCollection")) {
      AggregationFmrc aggc = new AggregationFmrc(newds, dimName, recheck);
      agg = aggc;

      // nested scanFmrc elements
      java.util.List<Element> scan2List = aggElem.getChildren("scanFmrc", ncNS);
      for (Element scanElem : scan2List) {
        String dirLocation = scanElem.getAttributeValue("location");
        if (dirLocation != null)
          dirLocation = AliasTranslator.translateAlias(dirLocation);
        String regexpPatternString = scanElem.getAttributeValue("regExp");
        String suffix = scanElem.getAttributeValue("suffix");
        String subdirs = scanElem.getAttributeValue("subdirs");
        String olderS = scanElem.getAttributeValue("olderThan");

        String runMatcher = scanElem.getAttributeValue("runDateMatcher");
        String forecastMatcher = scanElem.getAttributeValue("forecastDateMatcher");
        String offsetMatcher = scanElem.getAttributeValue("forecastOffsetMatcher");

        // possible relative location
        dirLocation = URLnaming.resolve(ncmlLocation, dirLocation);

        if (dirLocation != null) {
          aggc.addDirectoryScanFmrc(dirLocation, suffix, regexpPatternString, subdirs, olderS, runMatcher,
              forecastMatcher, offsetMatcher);
        }

        if ((cancelTask != null) && cancelTask.isCancel())
          return agg;
        if (debugAggDetail)
          System.out.println(" debugAgg: nested dirLocation = " + dirLocation);
      }

      // add explicit files to the agg (i.e. not from a scanned directory)
      Map<String, String> realLocationRunTimeMap = new HashMap<>();
      List<String> realLocationList = new ArrayList<>();
      java.util.List<Element> ncList = aggElem.getChildren("netcdf", ncNS);
      for (Element netcdfElemNested : ncList) {
        String location = netcdfElemNested.getAttributeValue("location");
        if (location == null)
          location = netcdfElemNested.getAttributeValue("url");
        if (location != null)
          location = AliasTranslator.translateAlias(location);

        String runTime = netcdfElemNested.getAttributeValue("coordValue");
        if (runTime == null) {
          Formatter f = new Formatter();
          f.format("runtime must be explicitly defined for each netcdf element using the attribute coordValue");
          log.error(f.toString());
        }

        String realLocation = URLnaming.resolveFile(ncmlLocation, location);
        realLocationRunTimeMap.put(realLocation, runTime);
        realLocationList.add(realLocation);

        if ((cancelTask != null) && cancelTask.isCancel())
          return aggc;
        if (debugAggDetail)
          System.out.println(" debugAgg: nested dataset = " + location);
      }

      if (!realLocationRunTimeMap.isEmpty()) {
        aggc.addExplicitFilesAndRunTimes(realLocationRunTimeMap);
      }
    } else {
      throw new IllegalArgumentException("Unknown aggregation type=" + type);
    }

    if (agg instanceof AggregationOuterDimension) {
      AggregationOuterDimension aggo = (AggregationOuterDimension) agg;

      String timeUnitsChange = aggElem.getAttributeValue("timeUnitsChange");
      if (timeUnitsChange != null)
        aggo.setTimeUnitsChange(timeUnitsChange.equalsIgnoreCase("true"));

      // look for variables that need to be aggregated (aggNew)
      java.util.List<Element> list = aggElem.getChildren("variableAgg", ncNS);
      for (Element vaggElem : list) {
        String varName = vaggElem.getAttributeValue("name");
        aggo.addVariable(varName);
      }

      // look for attributes to promote to variables
      list = aggElem.getChildren("promoteGlobalAttribute", ncNS);
      for (Element gattElem : list) {
        String varName = gattElem.getAttributeValue("name");
        String orgName = gattElem.getAttributeValue("orgName");
        aggo.addVariableFromGlobalAttribute(varName, orgName);
      }

      // look for attributes to promote to variables
      list = aggElem.getChildren("promoteGlobalAttributeCompose", ncNS);
      for (Element gattElem : list) {
        String varName = gattElem.getAttributeValue("name");
        String format = gattElem.getAttributeValue("format");
        String orgName = gattElem.getAttributeValue("orgName");
        aggo.addVariableFromGlobalAttributeCompose(varName, format, orgName);
      }

      // look for variable to cache
      list = aggElem.getChildren("cacheVariable", ncNS);
      for (Element gattElem : list) {
        String varName = gattElem.getAttributeValue("name");
        aggo.addCacheVariable(varName, null);
      }
    }

    // nested netcdf elements
    java.util.List<Element> ncList = aggElem.getChildren("netcdf", ncNS);
    for (Element netcdfElemNested : ncList) {
      String location = netcdfElemNested.getAttributeValue("location");
      if (location == null)
        location = netcdfElemNested.getAttributeValue("url");
      if (location != null)
        location = AliasTranslator.translateAlias(location);

      String id = netcdfElemNested.getAttributeValue("id");
      String ncoords = netcdfElemNested.getAttributeValue("ncoords");
      String coordValueS = netcdfElemNested.getAttributeValue("coordValue");
      String sectionSpec = netcdfElemNested.getAttributeValue("section");

      // must always open through a NcML reader, in case the netcdf element modifies the dataset
      NcmlElementReader reader = new NcmlElementReader(ncmlLocation, location, netcdfElemNested);
      String cacheName = (location != null) ? location : ncmlLocation;
      cacheName += "#" + netcdfElemNested.hashCode(); // need a unique name, in case file has been modified by ncml

      String realLocation = URLnaming.resolveFile(ncmlLocation, location);

      agg.addExplicitDataset(cacheName, realLocation, id, ncoords, coordValueS, sectionSpec, reader);

      if ((cancelTask != null) && cancelTask.isCancel())
        return agg;
      if (debugAggDetail)
        System.out.println(" debugAgg: nested dataset = " + location);
    }

    // nested scan elements
    java.util.List<Element> dirList = aggElem.getChildren("scan", ncNS);
    for (Element scanElem : dirList) {
      String dirLocation = scanElem.getAttributeValue("location");
      if (dirLocation == null)
        throw new IllegalArgumentException("scan element must have location attribute");

      dirLocation = AliasTranslator.translateAlias(dirLocation);

      String regexpPatternString = scanElem.getAttributeValue("regExp");
      String suffix = scanElem.getAttributeValue("suffix");
      String subdirs = scanElem.getAttributeValue("subdirs");
      String olderS = scanElem.getAttributeValue("olderThan");

      String dateFormatMark = scanElem.getAttributeValue("dateFormatMark");
      Set<NetcdfDataset.Enhance> enhanceMode = NetcdfDataset.parseEnhanceMode(scanElem.getAttributeValue("enhance"));

      // possible relative location
      dirLocation = URLnaming.resolve(ncmlLocation, dirLocation);

      // can embed a full-blown crawlableDatasetImpl element
      Element cdElement = scanElem.getChild("crawlableDatasetImpl", ncNS); // ok if null
      agg.addDatasetScan(cdElement, dirLocation, suffix, regexpPatternString, dateFormatMark, enhanceMode, subdirs,
          olderS);

      if ((cancelTask != null) && cancelTask.isCancel())
        return agg;
      if (debugAggDetail)
        System.out.println(" debugAgg: nested dirLocation = " + dirLocation);
    }

    // experimental
    Element collElem = aggElem.getChild("collection", ncNS);
    if (collElem != null)
      agg.addCollection(collElem.getAttributeValue("spec"), collElem.getAttributeValue("olderThan"));

    /*
     *
     * <xsd:choice minOccurs="0" maxOccurs="unbounded">
     * <xsd:element ref="group"/>
     * <xsd:element ref="dimension"/>
     * <xsd:element ref="variable"/>
     * <xsd:element ref="attribute"/>
     * <xsd:element ref="remove"/>
     * </xsd:choice>
     */
    boolean needMerge = !aggElem.getChildren("attribute", ncNS).isEmpty();
    if (!needMerge)
      needMerge = !aggElem.getChildren("variable", ncNS).isEmpty();
    if (!needMerge)
      needMerge = !aggElem.getChildren("dimension", ncNS).isEmpty();
    if (!needMerge)
      needMerge = !aggElem.getChildren("group", ncNS).isEmpty();
    if (!needMerge)
      needMerge = !aggElem.getChildren("remove", ncNS).isEmpty();
    if (needMerge)
      agg.setModifications(aggElem);

    return agg;
  }

  private class NcmlElementReader implements ucar.nc2.util.cache.FileFactory {
    private Element netcdfElem;
    private String ncmlLocation;
    private String location;

    NcmlElementReader(String ncmlLocation, String location, Element netcdfElem) {
      this.ncmlLocation = ncmlLocation;
      this.location = location;
      this.netcdfElem = netcdfElem;
    }

    public NetcdfFile open(DatasetUrl cacheName, int buffer_size, CancelTask cancelTask, Object spiObject)
        throws IOException {
      if (debugAggDetail)
        System.out.println(" NcmlElementReader open nested dataset " + cacheName);
      NetcdfFile result = _readNcML(ncmlLocation, location, netcdfElem, cancelTask);
      result.setLocation(ncmlLocation + "#" + location);
      return result;
    }
  }

  /////////////////////////////////////////////
  // command procesing

  private void cmdRemove(Group g, String type, String name) {
    boolean err = false;
    switch (type) {
      case "dimension":
        Dimension dim = g.findDimension(name);
        if (dim != null) {
          g.remove(dim);
          if (debugCmd)
            System.out.println("CMD remove " + type + " " + name);
        } else
          err = true;

        break;
      case "variable":
        Variable v = g.findVariableLocal(name);
        if (v != null) {
          g.remove(v);
          if (debugCmd)
            System.out.println("CMD remove " + type + " " + name);
        } else
          err = true;

        break;
      case "attribute":
        Attribute a = g.findAttribute(name);
        if (a != null) {
          g.remove(a);
          if (debugCmd)
            System.out.println("CMD remove " + type + " " + name);
        } else
          err = true;
        break;
    }

    if (err) {
      Formatter f = new Formatter();
      f.format("CMD remove %s CANT find %s location %s%n", type, name, location);
      log.info(f.toString());
    }
  }

  private void cmdRemove(Variable v, String type, String name) {
    boolean err = false;

    if (type.equals("attribute")) {
      ucar.nc2.Attribute a = v.findAttribute(name);
      if (a != null) {
        v.remove(a);
        if (debugCmd)
          System.out.println("CMD remove " + type + " " + name);
      } else
        err = true;

    } else if (type.equals("variable") && v instanceof Structure) {
      Structure s = (Structure) v;
      Variable nested = s.findVariable(name);
      if (nested != null) {
        s.removeMemberVariable(nested);
        if (debugCmd)
          System.out.println("CMD remove " + type + " " + name);
      } else
        err = true;

    }

    if (err) {
      Formatter f = new Formatter();
      f.format("CMD remove %s CANT find %s location %s%n", type, name, location);
      log.info(f.toString());
    }
  }

  ///////////////////////////////////////////////////////////////////////////////////

  /*
   * Read an NcML file and write an equivalent NetcdfFile to a physical file, using Netcdf-3 file format.
   *
   * @param ncmlLocation read this NcML file
   *
   * @param fileOutName write to this local file
   *
   * @throws IOException on write error
   *
   * @see ucar.nc2.FileWriter2
   *
   * public static void writeNcMLToFile(String ncmlLocation, String fileOutName) throws IOException {
   * NetcdfFile ncd = NetcdfDataset.acquireFile(ncmlLocation, null);
   *
   * FileWriter2 writer = new FileWriter2(ncd, fileOutName, NetcdfFileWriter.Version.netcdf3, null);
   * NetcdfFile result = writer.write();
   * result.close();
   * ncd.close();
   * }
   */

  /**
   * Read an NcML and write an equivalent NetcdfFile to a physical file, using Netcdf-3 file format.
   * The NcML may have a referenced dataset in the location URL, in which case the underlying data
   * (modified by the NcML) is written to the new file. If the NcML does not have a referenced dataset,
   * then the new file is filled with fill values, like ncgen.
   *
   * @param ncml read NcML from this input stream
   * @param fileOutName write to this local file
   * @see ucar.nc2.FileWriter2
   */
  public static void writeNcMLToFile(InputStream ncml, String fileOutName) throws IOException {
    writeNcMLToFile(ncml, fileOutName, NetcdfFileWriter.Version.netcdf3, null);
  }

  /**
   * Read an NcML and write an equivilent NetcdfFile to a physical file, using Netcdf-3 file format.
   * The NcML may have a referenced dataset in the location URL, in which case the underlying data
   * (modified by the NcML) is written to the new file. If the NcML does not have a referenced dataset,
   * then the new file is filled with fill values, like ncgen.
   *
   * @param ncml read NcML from this input stream
   * @param fileOutName write to this local file
   * @param version kind of netcdf file
   * @param chunker optional chunking (netcdf4 only)
   */
  public static void writeNcMLToFile(InputStream ncml, String fileOutName, NetcdfFileWriter.Version version,
      Nc4Chunking chunker) throws IOException {
    NetcdfDataset ncd = NcMLReader.readNcML(ncml, null);
    FileWriter2 writer = new FileWriter2(ncd, fileOutName, version, chunker);
    NetcdfFile result = writer.write();
    result.close();
    ncd.close();
  }

}


text:
<!doctype html>

    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
    <title>Drop – Browser Demo</title>
    <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
    <link rel="icon" href="http://static.hubspot.com/favicon.ico">

    <link rel="stylesheet" href="../../dist/css/tether-theme-arrows-dark.css" />

    <link rel="stylesheet" href="./css/browser-demo.css" />


text:
var vows = require("vows"),
    load = require("../load"),
    assert = require("assert");

var suite = vows.describe("mpld3.Figure");

suite.addBatch({
    "Figure": {
        topic: load("core/figure").document(),
        "A simple figure": {
            topic: function(mpld3) {
                var props = {
                    width: 400,
                    height: 300,
                    plugins: []
                };
                var figure = new mpld3.Figure("chart", props);
                figure.draw();
                return figure;
            },
            "has the expected id": function(figure){
                assert.equal(figure.figid, "chart");
            },
            "has the expected width and height": function(figure){
                assert.equal(figure.props.width, 400);
                assert.equal(figure.props.height, 300);
            }
        }
    }
});

suite.export(module);


text:
'use strict';

var path = require('path');
var zlib = require('zlib');

// Helper functions

function hasProcessFlag(flag) {
  return process.argv.join('').indexOf(flag) > -1;
}

function gzipMaxLevel(buffer, callback) {
  return zlib['gzip'](buffer, {level: 9}, callback);
}

function root(args) {
  args = Array.prototype.slice.call(arguments, 0);
  return path.join.apply(path, [__dirname].concat(args));
}

function rootNode(args) {
  args = Array.prototype.slice.call(arguments, 0);
  return root.apply(path, ['node_modules'].concat(args));
}

function prependExt(extensions, args) {
  args = args || [];
  if (!Array.isArray(args)) { args = [args]; }
  return extensions.reduce(function(memo, val) {
    return memo.concat(val, args.map(function(prefix) {
      return prefix + val;
    }));
  }, ['']);
}

exports.hasProcessFlag = hasProcessFlag;
exports.gzipMaxLevel = gzipMaxLevel;
exports.root = root;
exports.rootNode = rootNode;
exports.prependExt = prependExt;
exports.prepend = prependExt;


text:
ACCEPTED

#### According to
International Plant Names Index

#### Published in
null

#### Original name
Helipterum stoveae D.A.Cooke

### Remarks
null


text:
package session_test

import (
	gofail "github.com/etcd-io/gofail/runtime"
	. "github.com/pingcap/check"
	"github.com/pingcap/tidb/util/testkit"
)

func (s *testSessionSuite) TestFailStatementCommit(c *C) {
	defer gofail.Disable("github.com/pingcap/tidb/session/mockStmtCommitError")

	tk := testkit.NewTestKitWithInit(c, s.store)
	tk.MustExec("create table t (id int)")
	tk.MustExec("begin")
	tk.MustExec("insert into t values (1)")
	gofail.Enable("github.com/pingcap/tidb/session/mockStmtCommitError", `return(true)`)
	tk.MustExec("insert into t values (2)")
	_, err := tk.Exec("commit")
	c.Assert(err, NotNil)
	tk.MustQuery(`select * from t`).Check(testkit.Rows())
}