text:
using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.Net;
using System.Threading;
using System.ComponentModel;
using System.Collections;
using System.Windows.Forms;
using System.Xml;
using System.Diagnostics;

using dp2Secure;

using DigitalPlatform.Xml;
using DigitalPlatform.Marc;
using DigitalPlatform.Text;
using DigitalPlatform.IO;
using DigitalPlatform.Core;

namespace DigitalPlatform.AmazonInterface
{
    /// <summary>
    /// 检索亚马逊服务器
    /// </summary>
    public class AmazonSearch : IDisposable
    {
        public const string API_VERSION = "2013-08-01"; // "2011-08-01";

        public int SleepTime = 2000;

        /// <summary>
        /// 最近一次访问服务器的时间
        /// </summary>
        public DateTime LastTime = DateTime.Now;
        /// <summary>
        /// 等待的间隙
        /// </summary>
        public event EventHandler Idle = null;

        // public MainForm MainForm = null;
        public string TempFileDir = "";

        // Implement IDisposable.
        // Do not make this method virtual.
        // A derived class should not be able to override this method.
        public void Dispose()
        {
            Dispose(true);
            // This object will be cleaned up by the Dispose method.
            // Therefore, you should call GC.SuppressFinalize to
            // take this object off the finalization queue
            // and prevent finalization code for this object
            // from executing a second time.
            GC.SuppressFinalize(this);
        }

        private bool disposed = false;

        // Dispose(bool disposing) executes in two distinct scenarios.
        // If disposing equals true, the method has been called directly
        // or indirectly by a user's code. Managed and unmanaged resources
        // can be disposed.
        // If disposing equals false, the method has been called by the
        // runtime from inside the finalizer and you should not reference
        // other objects. Only unmanaged resources can be disposed.
        private void Dispose(bool disposing)
        {
            // Check to see if Dispose has already been called.
            if (!this.disposed)
            {
                // If disposing equals true, dispose all managed
                // and unmanaged resources.
                if (disposing)
                {
                    // Dispose managed resources.
                    this.eventComplete.Dispose();
                }

                // 删除临时文件
                DeleteTempFile();
            }
            disposed = true;
        }

#if NO
        // TODO: 容易造成 mem leak。建议用 Dispose() 改写
        ~AmazonSearch()
        {
            Dispose(false);
        }
#endif

        // 构造检索式
        int BuildQueryString(
            string strWord,
            string strFrom,
            string strMatch,
            out string strText,
            out string strError)
        {
            strError = "";
            strText = "";

            if (string.IsNullOrEmpty(strWord) == true)
            {
                strError = "检索词不能为空";
                return -1;
            }

            // TODO: 检查 strFrom 和 strMatch 的合法性

            if (strMatch == "[default]")
                strMatch = "";

            strText = strFrom + strMatch;
            if (strText.IndexOf(":") == -1)
                strText += ": ";

            strText += " " + strWord;
            return 0;
        }

        // private const string DESTINATION = "webservices.amazon.cn";  // "ecs.amazonaws.com";

        // 准备用于检索的 URL
        int GetOneLineSearchRequestUrl(
            string strServerUrl,
            string strWord,
            string strFrom,
            string strMatch,
            bool bUseFullElementSet,
            out string strUrl,
            out string strError)
        {
            strError = "";
            strUrl = "";

            if (string.IsNullOrEmpty(strWord) == true)
            {
                strError = "strWord 参数值不能为空";
                return -1;
            }

            string strText = "";
            // 构造检索式
            int nRet = BuildQueryString(
            strWord,
            strFrom,
            strMatch,
            out strText,
            out strError);
            if (nRet == -1)
                return -1;

            AmazonSignedRequestHelper helper = new AmazonSignedRequestHelper(
//MY_AWS_ACCESS_KEY_ID,
//MY_AWS_SECRET_KEY,
strServerUrl);

            IDictionary<string, string> parameters = new Dictionary<string, String>();

            parameters["Service"] = "AWSECommerceService";
            parameters["Version"] = API_VERSION;    // "2011-08-01";
            parameters["Operation"] = "ItemSearch";
            parameters["SearchIndex"] = "Books";
            parameters["Power"] = strText;

            if (bUseFullElementSet == true)
                parameters["ResponseGroup"] = "Large";
            else
                parameters["ResponseGroup"] = "Small";

#if TTT
            parameters["AssociateTag"] = ASSOCIATEKEY;
#endif

            m_searchParameters = parameters;
            // m_strCurrentSearchedServer = this.CurrentServer;

            strUrl = helper.Sign(parameters);
            return 0;
        }

        // 准备用于获得下一批浏览记录的请求的 URL
        int GetNextRequestUrl(
            string strServerUrl,
            out string strUrl,
            out string strError)
        {
            strError = "";
            strUrl = "";

            AmazonSignedRequestHelper helper = new AmazonSignedRequestHelper(
//MY_AWS_ACCESS_KEY_ID,
//MY_AWS_SECRET_KEY,
strServerUrl);

            if (this.m_nCurrentPageNo == -1)
            {
                strError = "m_nCurrentPageNo 尚未初始化";
                return -1;
            }

            // ItemPage URL 参数的值是从 1 开始计算的
            m_searchParameters["ItemPage"] = (this.m_nCurrentPageNo + 1 + 1).ToString();

            strUrl = helper.Sign(m_searchParameters);
            return 0;
        }

        public const string NAMESPACE = "http://webservices.amazon.com/AWSECommerceService/2013-08-01";

        WebClient webClient = null;
        public string TempFilename = "";

        // 准备临时文件名
        void PrepareTempFile()
        {
            DeleteTempFile();

            // 如果以前有临时文件名,就直接沿用
            if (string.IsNullOrEmpty(this.TempFilename) == true)
            {
                // this.TempFilename = Path.Combine(this.MainForm.DataDir, "~webclient_response_" + Guid.NewGuid().ToString());
                Debug.Assert(string.IsNullOrEmpty(this.TempFileDir) == false, "");
                this.TempFilename = Path.Combine(this.TempFileDir, "~webclient_response_" + Guid.NewGuid().ToString());
            }

            try
            {
            }
            catch
            {
            }
            // 这一句移到 try ... catch 以外,是为了观察测试抛出异常的情况。如果测试完成后,需要放回里面去
            File.Delete(this.TempFilename);
        }

        void DeleteTempFile()
        {
            // DisposeWebClients();

            // 删除临时文件
            if (string.IsNullOrEmpty(this.TempFilename) == false)
            {
                try
                {
                }
                catch
                {
                }
                // 这一句移到 try ... catch 以外,是为了观察测试抛出异常的情况。如果测试完成后,需要放回里面去
                File.Delete(this.TempFilename);
                this.TempFilename = "";
            }
        }

        AutoResetEvent eventComplete = new AutoResetEvent(false);
        bool m_bError = false;   // 最近一次异步操作是否因报错结束
        Exception m_exception = null;

        public Exception Exception
        {
            get
            {
                return this.m_exception;
            }
        }

        string m_strError = ""; // 异步操作中用于保存出错信息

        bool _userFullElementSet = true;

#if NO
        List<WebClient> _webClients = new List<WebClient>();

        void DisposeWebClients()
        {
            foreach (WebClient webClient in _webClients)
            {
                if (webClient != null)
                {
                    webClient.DownloadFileCompleted -= new AsyncCompletedEventHandler(webClient_MultiLineDownloadFileCompleted);
                    webClient.Dispose();
                }
            }

            _webClients.Clear();
        }
#endif

        // 多行检索中的一行检索
        // return:
        //      -1  出错
        //      0   成功
        public int Search(
            string strServerUrl,
            string strWord,
            string strFrom,
            string strMatch,
            bool bUseFullElementSet,
            out string strError)
        {
            strError = "";
            int nRet = 0;

            _userFullElementSet = bUseFullElementSet;

            string strUrl = "";
            nRet = GetOneLineSearchRequestUrl(
                strServerUrl,
                strWord,
                strFrom,
                strMatch,
                bUseFullElementSet,
                out strUrl,
                out strError);
            if (nRet == -1)
                return -1;

            // this.m_multiSearchInfo.CurrentWord = strLine;

            // m_reloadInfo = null;

            this.PrepareTempFile();

            this.m_bError = false;
            this.m_exception = null;
#if NO
            if (this.webClient != null)
            {
                webClient.DownloadFileCompleted -= new AsyncCompletedEventHandler(webClient_MultiLineDownloadFileCompleted);
                this.webClient.Dispose();
                this.webClient = null;
            }
#endif
            webClient = new WebClient();

            // _webClients.Add(webClient);

            webClient.DownloadFileCompleted += new AsyncCompletedEventHandler(webClient_MultiLineDownloadFileCompleted);
            // webClient.DownloadProgressChanged += new DownloadProgressChangedEventHandler(webClient_DownloadProgressChanged);
            try
            {
                Delay();

                this.LastTime = DateTime.Now;
                webClient.DownloadFileAsync(new Uri(strUrl, UriKind.Absolute),
                    this.TempFilename, null);
            }
            catch (Exception ex)
            {
                this.m_exception = ex;
                strError = ExceptionUtil.GetAutoText(ex);
                this.m_bError = true;
                goto ERROR1;
            }

            // 等待检索结束
            bool bError = WaitSearchFinish();
            if (bError == true)
            {
                strError = this.m_strError;
                goto ERROR1;
            }

            {
                AsyncCompletedEventArgs e = _end_e;

                if (e == null || e.Cancelled == true)
                {
                    strError = "请求被取消";
                    goto ERROR1;
                }

                if (e != null && e.Error != null)
                {
                    strError = "请求过程发生错误: " + ExceptionUtil.GetExceptionMessage(e.Error);
                    this.m_exception = e.Error;
                    goto ERROR1;
                }
            }

            // 如果要求每行的检索命中装入大于 10 条,需要在这里获取后面几批的浏览结果
            return 0;
            ERROR1:
            // 迫使更换临时文件
            DeleteTempFile();   // 2015/8/4
            return -1;
        }

        // 休眠一段时间
        public void Delay()
        {
            TimeSpan delta = new TimeSpan(0, 0, 0, 0, SleepTime);
            TimeSpan delta1 = DateTime.Now - this.LastTime;
            if (this.Idle == null)
            {
                if (delta1 < delta)
                {
                    Thread.Sleep((int)((delta - delta1).TotalMilliseconds));
                }
            }
            else
            {
                while (DateTime.Now - this.LastTime < delta)
                {
                    if (this.Idle != null)
                    {
                        this.Idle(this, new EventArgs());
                    }
                    Thread.Sleep(10);
                }
            }
        }

        public bool HasNextBatch()
        {
            if (m_nTotalPages == 0)
                return false;
            if (m_nCurrentPageNo < m_nTotalPages - 1)
                return true;
            return false;
        }

        public int HitCount
        {
            get
            {
                return (int)this.m_nTotalResults;
            }
        }

        // 获得后一批结果
        public int NextBatch(
    string strServerUrl,
    out string strError)
        {
            strError = "";
            int nRet = 0;

            string strUrl = "";
            nRet = GetNextRequestUrl(
                strServerUrl,
                out strUrl,
                out strError);
            if (nRet == -1)
                return -1;

            Debug.Assert(string.IsNullOrEmpty(strUrl) == false, "");

            this.PrepareTempFile();

            this.m_bError = false;
            this.m_exception = null;

            webClient = new WebClient();

            webClient.DownloadFileCompleted += new AsyncCompletedEventHandler(webClient_MultiLineDownloadFileCompleted);
            // webClient.DownloadProgressChanged += new DownloadProgressChangedEventHandler(webClient_DownloadProgressChanged);
            try
            {
                Delay();

                this.LastTime = DateTime.Now;
                webClient.DownloadFileAsync(new Uri(strUrl, UriKind.Absolute),
                    this.TempFilename, null);
            }
            catch (Exception ex)
            {
                this.m_exception = ex;
                strError = ExceptionUtil.GetAutoText(ex);
                this.m_bError = true;
                goto ERROR1;
            }

            // 等待检索结束
            bool bError = WaitSearchFinish();
            if (bError == true)
            {
                strError = this.m_strError;
                goto ERROR1;
            }

            {
                AsyncCompletedEventArgs e = _end_e;

                if (e == null || e.Cancelled == true)
                {
                    strError = "请求被取消";
                    goto ERROR1;
                }

                if (e != null && e.Error != null)
                {
                    strError = "请求过程发生错误: " + ExceptionUtil.GetExceptionMessage(e.Error);
                    this.m_exception = e.Error;
                    goto ERROR1;
                }
            }

            return 0;
            ERROR1:
            // 迫使更换临时文件
            DeleteTempFile();   // 2015/8/4
            return -1;
        }

        /// <summary>
        /// 超时值。单位为毫秒
        /// </summary>
        public int Timeout = 5000;

        // TODO: 加入一旦窗口关闭就跳出循环的逻辑
        // return:
        //      true    已经报错
        //      false   正常
        bool WaitSearchFinish()
        {
            int nTimeCount = 0; // 如果为 -1 表示已经 Cancel
            while (true)
            {
                if (this.Idle != null)
                {
                    this.Idle(this, new EventArgs());
                }

                if (nTimeCount > 0 && nTimeCount > this.Timeout)
                {
                    webClient.CancelAsync();
#if NO
                    nTimeCount = -1;
                    this.m_strError = "超时";
                    this.m_bError = true;
                    break;  // 2015/5/22
#endif
                }
                // Application.DoEvents();
                if (eventComplete.WaitOne(100) == true)
                    break;
                if (nTimeCount >= 0)
                    nTimeCount += 100;
            }

            return this.m_bError;
        }

        // 终止检索
        public void CancelSearch()
        {
            if (webClient != null)
                webClient.CancelAsync();
        }

        AsyncCompletedEventArgs _end_e = null;

        void webClient_MultiLineDownloadFileCompleted(object sender, AsyncCompletedEventArgs e)
        {
#if NO
            Delegate_EndSearch d = new Delegate_EndSearch(EndMultiLineSearch);
            object[] args = new object[] { e };
            this.MainForm.Invoke(d, args);
#endif
            _end_e = e;

            if (this.eventComplete != null)
                this.eventComplete.Set();
        }

        // 重新装载浏览记录的任务信息
        class ReloadTaskInfo
        {
            public string ElementSet = "";
            public List<ListViewItem> TotalItems = null;
            public List<ListViewItem> CurrentItems = null;
            public int StartIndex = 0;  // 开始的偏移
            public bool Cancel = false; // 是否中途放弃,或者因为出错而需要中断
        }
        ReloadTaskInfo m_reloadInfo = null;
        long m_nTotalResults = 0;    // 当前检索命中的结果数
        long m_nTotalPages = 0;     // 命中结果的总页数
        int m_nCurrentPageNo = -1;  // 当前已经装入的最后一个页号
        IDictionary<string, string> m_searchParameters = null;  // 当前检索参数

        void ClearResultSetParameters()
        {
            this.m_nCurrentPageNo = -1; // 表示这是第一次检索
            this.m_nTotalResults = 0;
            this.m_nTotalPages = 0;
            this.m_searchParameters = null;
            //this.m_strCurrentSearchedServer = "";
        }

        // return:
        //      -1  出错
        //      0   正常
        //      1   希望停止
        public delegate int Delegate_appendBrowseLine(string strRecPath,
            string strRecord,
            object param,
            bool bAutoSetFocus,
            out string strError);

        // return:
        //      -1  出错
        //      >=0 命中的记录数
        public int LoadBrowseLines(Delegate_appendBrowseLine appendBrowseLineProc,
            object param,
            bool bAutoSetFocus,
            out string strError)
        {
            strError = "";

            XmlDocument doc = new XmlDocument();
            doc.Load(this.TempFilename);

            XmlNamespaceManager nsmgr = new XmlNamespaceManager(new NameTable());
            nsmgr.AddNamespace("amazon", NAMESPACE);

            XmlNodeList errors = doc.DocumentElement.SelectNodes("amazon:Items/amazon:Request/amazon:Errors/amazon:Error", nsmgr);
            if (errors.Count > 0)
            {
                string strCode = DomUtil.GetElementText(errors[0], "amazon:Code", nsmgr);
                string strMessage = DomUtil.GetElementText(errors[0], "amazon:Message", nsmgr);
                if (strCode == "AWS.ECommerceService.NoExactMatches")
                {
                    strError = strCode;
                    goto ERROR1;
                }
                strError = strMessage;
                goto ERROR1;
            }

            int nHitCount = 0;

            if (this.m_reloadInfo == null)
            {
                // Items/TotalResults
                string strTotalResults = DomUtil.GetElementText(doc.DocumentElement,
                    "amazon:Items/amazon:TotalResults", nsmgr);
                Int64.TryParse(strTotalResults, out this.m_nTotalResults);

                // Items/TotalPages
                string strTotalPages = DomUtil.GetElementText(doc.DocumentElement,
                    "amazon:Items/amazon:TotalPages", nsmgr);
                Int64.TryParse(strTotalPages, out this.m_nTotalPages);

                nHitCount = (int)Math.Min(m_nTotalResults, 10);

#if NO
                // TODO: 显示单行命中的消息
                if (m_nTotalResults > 0)
                {
                    int nHitCount = (int)Math.Min(m_nTotalResults, 10);
                    this.m_multiSearchInfo.HitWords.Add(this.m_multiSearchInfo.CurrentWord + "\t" + nHitCount.ToString());
                    this.m_multiSearchInfo.HitCount += nHitCount;
                }
                else
                    this.m_multiSearchInfo.NotHitWords.Add(this.m_multiSearchInfo.CurrentWord);
#endif

                // this.amazonSimpleQueryControl_simple.Comment = "命中记录:\t" + m_nTotalResults.ToString();
            }

            int nRet = LoadResults(doc,
                // _userFullElementSet,
                appendBrowseLineProc,
                param,
                bAutoSetFocus,
                out strError);
            if (nRet == -1)
                goto ERROR1;

            if (this.m_reloadInfo == null)
            {
                this.m_nCurrentPageNo++;    // 第一次 从 -1 ++ 正好等于 0
            }
            return nHitCount;
            ERROR1:
            if (this.m_reloadInfo != null)
                this.m_reloadInfo.Cancel = true;

            this.DeleteTempFile();

            if (strError == "AWS.ECommerceService.NoExactMatches")
            {
                // TODO: 显示单行没有命中的消息
                ////this.m_multiSearchInfo.NotHitWords.Add(this.m_multiSearchInfo.CurrentWord);
                return 0;
            }
            // TODO: 显示单行检索出错的消息
            // this.amazonSimpleQueryControl_simple.Comment = "检索发生错误:\r\n" + strError;
            this.m_bError = true;
            this.m_strError = strError;
            return -1;
        }

#if NO
        delegate void Delegate_EndSearch(AsyncCompletedEventArgs e);
        void EndMultiLineSearch(AsyncCompletedEventArgs e)
        {
            string strError = "";
            if (e == null || e.Cancelled == true)
            {
                strError = "请求被取消";
                goto ERROR1;
            }

            if (e != null && e.Error != null)
            {
                strError = "请求过程发生错误: " + ExceptionUtil.GetExceptionMessage(e.Error);
                this.m_exception = e.Error; // 2013/4/25
                goto ERROR1;
            }

            XmlDocument doc = new XmlDocument();
            doc.Load(this.TempFilename);

            XmlNamespaceManager nsmgr = new XmlNamespaceManager(new NameTable());
            nsmgr.AddNamespace("amazon", NAMESPACE);

            XmlNodeList errors = doc.DocumentElement.SelectNodes("amazon:Items/amazon:Request/amazon:Errors/amazon:Error", nsmgr);
            if (errors.Count > 0)
            {
                string strCode = DomUtil.GetElementText(errors[0], "amazon:Code", nsmgr);
                string strMessage = DomUtil.GetElementText(errors[0], "amazon:Message", nsmgr);
                if (strCode == "AWS.ECommerceService.NoExactMatches")
                {
                    strError = strCode;
                    goto ERROR1;
                }
                strError = strMessage;
                goto ERROR1;
            }

            if (this.m_reloadInfo == null)
            {
                // Items/TotalResults
                string strTotalResults = DomUtil.GetElementText(doc.DocumentElement,
                    "amazon:Items/amazon:TotalResults", nsmgr);
                Int64.TryParse(strTotalResults, out this.m_nTotalResults);

                // Items/TotalPages
                string strTotalPages = DomUtil.GetElementText(doc.DocumentElement,
                    "amazon:Items/amazon:TotalPages", nsmgr);
                Int64.TryParse(strTotalPages, out this.m_nTotalPages);

#if NO
                // TODO: 显示单行命中的消息
                if (m_nTotalResults > 0)
                {
                    int nHitCount = (int)Math.Min(m_nTotalResults, 10);
                    this.m_multiSearchInfo.HitWords.Add(this.m_multiSearchInfo.CurrentWord + "\t" + nHitCount.ToString());
                    this.m_multiSearchInfo.HitCount += nHitCount;
                }
                else
                    this.m_multiSearchInfo.NotHitWords.Add(this.m_multiSearchInfo.CurrentWord);
#endif

                // this.amazonSimpleQueryControl_simple.Comment = "命中记录:\t" + m_nTotalResults.ToString();
            }

            int nRet = LoadResults(doc, _userFullElementSet, out strError);
            if (nRet == -1)
                goto ERROR1;

            if (this.m_reloadInfo == null)
            {
                this.m_nCurrentPageNo++;    // 第一次 从 -1 ++ 正好等于 0
            }
            return;
        ERROR1:
            if (this.m_reloadInfo != null)
                this.m_reloadInfo.Cancel = true;

            this.DeleteTempFile();

            if (strError == "AWS.ECommerceService.NoExactMatches")
            {
                // TODO: 显示单行没有命中的消息
                ////this.m_multiSearchInfo.NotHitWords.Add(this.m_multiSearchInfo.CurrentWord);
                return;
            }
            // TODO: 显示单行检索出错的消息
            // this.amazonSimpleQueryControl_simple.Comment = "检索发生错误:\r\n" + strError;
            this.m_bError = true;
            this.m_strError = strError;
        }
#endif

        // 装入检索命中记录到浏览列表中
        int LoadResults(XmlDocument response_dom,
            // bool bDisplayInfo,
            Delegate_appendBrowseLine appendBrowseLineProc,
            object param,
            bool bAutoSetFocus,
            out string strError)
        {
            strError = "";
            int nRet = 0;

            // string strElementSet = "";

            // 建立 ASIN --> ListViewItem 对照表
            Hashtable table = new Hashtable();
#if NO
            if (this.m_reloadInfo != null)
            {
                foreach (ListViewItem item in this.m_reloadInfo.CurrentItems)
                {
                    table[item.Text] = item;
                }
                strElementSet = this.m_reloadInfo.ElementSet;
            }
            else
            {
                if (bUseFullElementSet == true)
                    strElementSet = "F";
                else
                    strElementSet = "B";
            }
#endif

            XmlNamespaceManager nsmgr = new XmlNamespaceManager(new NameTable());
            nsmgr.AddNamespace("amazon", NAMESPACE);

            XmlNodeList items = response_dom.DocumentElement.SelectNodes("amazon:Items/amazon:Item", nsmgr);

            foreach (XmlNode item in items)
            {
                var element = item as XmlElement;

                List<string> cols = null;
                string strASIN = "";
                string strCoverImageUrl = "";
                nRet = ParseItemXml(element,
                    nsmgr,
                    out strASIN,
                    out strCoverImageUrl,
                    out cols,
                    out strError);
                if (nRet == -1)
                    return -1;
                nRet = appendBrowseLineProc("", element.OuterXml, param, bAutoSetFocus, out strError);
                if (nRet == -1)
                    return -1;
                if (nRet == 1)
                    break;

#if NO
                if (info.EelementSet == "B")
                    listitem.ImageIndex = BROWSE_TYPE_BRIEF;
                else
                    listitem.ImageIndex = BROWSE_TYPE_FULL;
#endif
            }

#if NO
            if (this.m_reloadInfo == null)
            {
                if (bDisplayInfo == true)
                {
                    this.amazonSimpleQueryControl_simple.Comment = "命中记录:\t" + m_nTotalResults.ToString()
                    + "\r\n已装入浏览记录:\t" + this.listView_browse.Items.Count.ToString();
                }
            }
            else
            {
                if (table.Count != 0)
                {
                    strError = "下列事项没有被重新装载 '" + Join(table.Keys) + "'";
                    return -1;
                }
            }
#endif

            return 0;
        }

        static string Join(ICollection list)
        {
            StringBuilder text = new StringBuilder();
            foreach (string s in list)
            {
                if (text.Length > 0)
                    text.Append(",");
                text.Append(s);
            }

            return text.ToString();
        }

        static string[] imagesize_names = new string[] {
            "SmallImage",
            "MediumImage",
            "LargeImage"};

        public static int GetImageUrl(XmlElement root,
    XmlNamespaceManager nsmgr,
    out Hashtable table,
    out string strError)
        {
            strError = "";
            table = new Hashtable();

            foreach (string s in imagesize_names)
            {
                List<ImageInfo> images = GetImageValues(root,
    nsmgr,
    "amazon:" + s);
                if (images.Count > 0)
                    table[s] = images[0];
            }
            return 0;
        }

        // 解析 <Item> 内的基本信息
        // parameters:
        //      strASIN
        //      cols    浏览列信息
        public static int ParseItemXml(XmlElement root,
            XmlNamespaceManager nsmgr,
            out string strASIN,
            out string strCoverImageUrl,
            out List<string> cols,
            out string strError)
        {
            strError = "";
            strASIN = "";
            strCoverImageUrl = "";
            cols = new List<string>();

            strASIN = DomUtil.GetElementText(root, "amazon:ASIN", nsmgr);

#if NO
            // ISBN
            cols.Add(
                GetFieldValues(root,
                nsmgr,
                "amazon:ItemAttributes/amazon:ISBN | amazon:ItemAttributes/amazon:ISSN",
                " ; ")
                );
#endif

            // title
            cols.Add(
                GetFieldValues(root,
                nsmgr,
                "amazon:ItemAttributes/amazon:Title",
                " ; ")
                );

            // author
            cols.Add(
    GetFieldValues(root,
    nsmgr,
    "amazon:ItemAttributes/amazon:Creator",
    " ; ")
    );
            // publisher
            cols.Add(
GetFieldValues(root,
nsmgr,
"amazon:ItemAttributes/amazon:Manufacturer",   // Publisher 在 Small 的时候没有提供
" ; ")
);
            // publish date
            cols.Add(
GetFieldValues(root,
nsmgr,
"amazon:ItemAttributes/amazon:PublicationDate", // PublicationDate 在 Small 的时候没有提供
" ; ")
);

#if NO
            // EAN
            cols.Add(
                GetFieldValues(root,
                nsmgr,
                "amazon:ItemAttributes/amazon:EAN",
                " ; ")
                );
#endif
            List<ImageInfo> images = GetImageValues(root,
nsmgr,
"amazon:MediumImage");
            if (images.Count > 0)
                strCoverImageUrl = images[0].Url;
            return 0;
        }

        // 获得一种字段的值。如果字段多次出现,用 strSep 符号连接
        static string GetFieldValues(XmlNode root,
            XmlNamespaceManager nsmgr,
            string strXPath,
            string strSep)
        {
            StringBuilder text = new StringBuilder();
            XmlNodeList nodes = root.SelectNodes(strXPath, nsmgr);
            foreach (XmlNode node in nodes)
            {
                if (text.Length > 0)
                    text.Append(strSep);
                text.Append(node.InnerText);
            }

            return text.ToString();
        }

        // 获得一种字段的值
        static List<string> GetFieldValues(XmlNode root,
            XmlNamespaceManager nsmgr,
            string strXPath)
        {
            List<string> results = new List<string>();
            XmlNodeList nodes = root.SelectNodes(strXPath, nsmgr);
            foreach (XmlNode node in nodes)
            {
                results.Add(node.InnerText.Trim());
            }

            return results;
        }

        // 获得价格字段的值
        static List<string> GetPriceValues(XmlNode root,
            XmlNamespaceManager nsmgr,
            string strXPath)
        {
            List<string> results = new List<string>();
            XmlNodeList nodes = root.SelectNodes(strXPath, nsmgr);
            foreach (XmlElement node in nodes)
            {
                results.Add(GetPriceValue(node, nsmgr));
            }

            return results;
        }

        /*
         * 日元比较特殊
        <ListPrice>
            <Amount>1404</Amount>
            <CurrencyCode>JPY</CurrencyCode>
            <FormattedPrice>¥ 1,404</FormattedPrice>
        </ListPrice>
         * 美元和人民币等货币,要把 Amount 元素的值除以 100
        <ListPrice>
            <Amount>3495</Amount>
            <CurrencyCode>USD</CurrencyCode>
            <FormattedPrice>$34.95</FormattedPrice>
        </ListPrice>
         * */
        static string GetPriceValue(XmlElement element,
            XmlNamespaceManager nsmgr)
        {
            XmlElement amount = element.SelectSingleNode("amazon:Amount", nsmgr) as XmlElement;
            XmlElement code = element.SelectSingleNode("amazon:CurrencyCode", nsmgr) as XmlElement;
            XmlElement formatted = element.SelectSingleNode("amazon:FormattedPrice", nsmgr) as XmlElement;

            string strText = "";
            if (code != null)
                strText += code.InnerText.Trim();

            // 2016/5/13
            // 优先用 FormattedPrice 元素
            if (formatted != null)
            {
                strText += PriceUtil.GetPurePrice(formatted.InnerText.Trim().Replace(",", ""));
                return strText;
            }

            if (amount != null)
            {
                string strAmount = amount.InnerText.Trim();
                if (string.IsNullOrEmpty(strAmount) == false)
                {
                    if (code.InnerText.Trim() != "JPY")
                    {
                        long v = 0;
                        if (long.TryParse(strAmount, out v) == false)
                        {
                            strText += " 数字 '" + strAmount + "' 格式错误";
                            return strText;
                        }

                        strAmount = (((decimal)v) / 100).ToString();
                        strText += strAmount;
                    }
                    else
                        strText += strAmount;
                }
            }
            return strText;
        }

        class Creator
        {
            public string Name = "";
            public string Role = "";
        }

        // 获得创建者字段的值
        static List<Creator> GetCreatorValues(XmlNode root,
            XmlNamespaceManager nsmgr,
            string strXPath)
        {
            List<Creator> results = new List<Creator>();
            XmlNodeList nodes = root.SelectNodes(strXPath, nsmgr);
            foreach (XmlElement node in nodes)
            {
                results.Add(GetCreatorValue(node, nsmgr));
            }

            return results;
        }

        static Creator GetCreatorValue(XmlElement element,
    XmlNamespaceManager nsmgr)
        {
            string strRole = element.GetAttribute("Role");

            if (strRole == "作者")
                strRole = "著";
            if (strRole == "译者")
                strRole = "译";

            Creator creator = new Creator();
            creator.Name = element.InnerText.Trim();
            creator.Role = strRole;

            return creator;
        }

        // 获得尺寸字段的值
        static List<string> GetHeightValues(XmlNode root,
            XmlNamespaceManager nsmgr,
            string strXPath)
        {
            List<string> results = new List<string>();
            XmlNodeList nodes = root.SelectNodes(strXPath, nsmgr);
            foreach (XmlElement node in nodes)
            {
                results.Add(GetHeightValue(node, nsmgr));
            }

            return results;
        }

        static string GetHeightValue(XmlElement element,
            XmlNamespaceManager nsmgr)
        {
            XmlElement height = element.SelectSingleNode("amazon:Length", nsmgr) as XmlElement;

            if (height != null)
            {
                string strUnits = height.GetAttribute("Units");
                string strValue = height.InnerText.Trim();

                if (string.IsNullOrEmpty(strValue) == false)
                {
                    double v = 0;
                    if (double.TryParse(strValue, out v) == false)
                    {
                        return " 数字 '" + strValue + "' 格式错误";
                    }

                    if (strUnits == "inches")
                        return Math.Ceiling(((double)v) * (double)2.54).ToString() + "cm";
                    if (strUnits == "hundredths-inches")
                        return Math.Ceiling(((double)v / (double)100) * (double)2.54).ToString() + "cm";

                    return strValue + strUnits;
                }
            }

            return "";
        }

        public class ImageInfo
        {
            public string Url = "";
            public string Size = "";
        }

        // 获得图像字段的值
        static List<ImageInfo> GetImageValues(XmlNode root,
            XmlNamespaceManager nsmgr,
            string strXPath)
        {
            List<ImageInfo> results = new List<ImageInfo>();
            XmlNodeList nodes = root.SelectNodes(strXPath, nsmgr);
            foreach (XmlElement node in nodes)
            {
                ImageInfo info = GetImageValue(node, nsmgr);
                if (info != null)
                    results.Add(info);
            }

            return results;
        }

        static string GetHeight(XmlElement element,
            XmlNamespaceManager nsmgr)
        {
            string strUnits = element.GetAttribute("Units");
            string strValue = element.InnerText.Trim();

            if (strUnits == "pixels")
                strUnits = "px";

            return strValue + strUnits;
        }

        static ImageInfo GetImageValue(XmlElement element,
            XmlNamespaceManager nsmgr)
        {
            XmlElement url = element.SelectSingleNode("amazon:URL", nsmgr) as XmlElement;
            XmlElement height = element.SelectSingleNode("amazon:Height", nsmgr) as XmlElement;
            XmlElement width = element.SelectSingleNode("amazon:Width", nsmgr) as XmlElement;

            ImageInfo info = new ImageInfo();

            if (url != null)
                info.Url = url.InnerText.Trim();

            // 2014/12/14
            // <URL>http://g-ec4.images-amazon.com/images/G/28/x-site/icons/no-img-sm._V192562228_._SL75_.gif</URL>
            if (info.Url.IndexOf("/no-img-") != -1)
                return null;

            if (height != null && width != null)
            {
                string strHeight = GetHeight(height, nsmgr);
                string strWidth = GetHeight(width, nsmgr);

                string strError = "";
                string strHeightValue = "";
                string strHeightUnit = "";
                int nRet = StringUtil.ParseUnit(strHeight,
    out strHeightValue,
    out strHeightUnit,
    out strError);
                if (nRet == -1)
                    strHeightValue = strHeight;

                string strWidthValue = "";
                string strWidthUnit = "";
                nRet = StringUtil.ParseUnit(strWidth,
    out strWidthValue,
    out strWidthUnit,
    out strError);
                if (nRet == -1)
                    strWidthValue = strWidth;

                if (strHeightUnit == strWidthUnit)
                    info.Size = strWidthValue + "X" + strHeightValue + strHeightUnit;    // 省略法
                else
                    info.Size = strWidth + "X" + strHeight;
            }
            return info;
        }

        /*
    <EditorialReviews>
        <EditorialReview>
            <Source>内容简介</Source>
            <Content>  这本由佛瑞斯特·卡特著的《少年小树之歌》讲述了:差不多在印第安紫罗兰开花以后,你会讶异地发现,拂上脸庞的风竟然如羽毛般温柔暖和,而且还带着一些泥土的气息,春天的脚步正渐渐地接近。<br>    接着闪电又来了,它击中了山顶的岩石,巨大的蓝色火球四处进落,照得整个天空一片闪动着的蓝。豆大的雨点稀落地从云端洒下,仿佛预告着接下来大得惊人的雨即将降临。<br>    夏天则是生命力量最澎湃的时刻,就像人在壮年一般。而当萧瑟的秋风吹起,我们觉得自己开始苍老,一股归乡的思绪充塞心头,有些人称这感觉为乡愁。万物销声匿迹的冬天就像我们躯体的死亡,但是当春天来临时,它们又会重生。</Content>
            <IsLinkSuppressed>0</IsLinkSuppressed>
        </EditorialReview>
    </EditorialReviews>
         * */
        // 获得价格字段的值
        // parameters:
        //      strXPath    EditorialReviews/EditorialReview
        //      strSourceType   "内容简介"
        static List<string> GetCommentValues(XmlNode root,
            XmlNamespaceManager nsmgr,
            string strXPath,
            string strSourceType)
        {
            List<string> results = new List<string>();
            XmlNodeList nodes = root.SelectNodes(strXPath, nsmgr);
            foreach (XmlElement node in nodes)
            {
                string strText = GetCommentValue(node, nsmgr, strSourceType);
                if (string.IsNullOrEmpty(strText) == false)
                    results.Add(strText);
            }

            return results;
        }

        static string GetCommentValue(XmlElement element,
            XmlNamespaceManager nsmgr,
            string strSourceType)
        {
            XmlElement source = element.SelectSingleNode("amazon:Source", nsmgr) as XmlElement;
            XmlElement content = element.SelectSingleNode("amazon:Content", nsmgr) as XmlElement;

            if (source == null)
                return "";
            if (source != null && source.InnerText.Trim() != strSourceType)
                return "";
            return content.InnerText.Trim();
        }

        // 将亚马逊 XML 格式转换为 UNIMARC 格式
        // parameters:
        //      strStyle    转换风格。!856 表示不包含 856 字段
        public static int AmazonXmlToUNIMARC(XmlNode root,
            string strStyle,
            out string strMARC,
            out string strError)
        {
            strMARC = "";
            strError = "";

            XmlNamespaceManager nsmgr = new XmlNamespaceManager(new NameTable());
            nsmgr.AddNamespace("amazon", AmazonSearch.NAMESPACE);

            MarcRecord record = new MarcRecord();

            // *** 001
            string strASIN = DomUtil.GetElementText(root, "amazon:ASIN", nsmgr);
            if (string.IsNullOrEmpty(strASIN) == false)
                record.ChildNodes.add(new MarcField("001", "ASIN:" + strASIN));

            // *** 010
            // ISBN
#if NO
            List<string> isbns = GetFieldValues(root,
                nsmgr,
                "amazon:ItemAttributes/amazon:ISBN");
            if (isbns.Count == 0)
                isbns = GetFieldValues(root,
                nsmgr,
                "amazon:ItemAttributes/amazon:EAN");
#endif
#if NO
            List<string> isbns = GetFieldValues(root,
    nsmgr,
    "amazon:ItemAttributes/amazon:EAN"
    );
            if (isbns.Count == 0)
                isbns = GetFieldValues(root,
                nsmgr,
                "amazon:ItemAttributes/amazon:ISBN");
#endif
            List<string> isbns = new List<string>();
            {
                List<string> temp = new List<string>();
                temp.AddRange(GetFieldValues(root,
    nsmgr,
    "amazon:ItemAttributes/amazon:ISBN"));
                temp.AddRange(GetFieldValues(root,
                    nsmgr,
                    "amazon:ItemAttributes/amazon:EAN"));
                // 从集合里面找 978 开头的
                foreach (string isbn in temp)
                {
                    if (isbn.StartsWith("978"))
                        isbns.Add(isbn);
                }

                // 实在没有的话,就用没有筛选时的内容
                if (isbns.Count == 0)
                    isbns.AddRange(temp);
            }

            // Binding
            List<string> bindings = GetFieldValues(root,
                nsmgr,
                "amazon:ItemAttributes/amazon:Binding");

            // 价格
            List<string> prices = GetPriceValues(root,
                nsmgr,
                "amazon:ItemAttributes/amazon:ListPrice");

            for (int i = 0; i < Math.Max(isbns.Count, prices.Count); i++)
            {
                string isbn = "";
                string binding = "";
                string price = "";

                if (i < isbns.Count)
                    isbn = isbns[i];
                if (i < bindings.Count)
                    binding = bindings[i];
                if (i < prices.Count)
                    price = prices[i];
                MarcField field = new MarcField("010", "  ");
                record.ChildNodes.add(field);

                if (string.IsNullOrEmpty(isbn) == false)
                    field.ChildNodes.add(new MarcSubfield("a", isbn));
                if (string.IsNullOrEmpty(binding) == false
                    && binding != "平装")
                    field.ChildNodes.add(new MarcSubfield("b", binding));
                if (string.IsNullOrEmpty(price) == false)
                    field.ChildNodes.add(new MarcSubfield("d", price));
            }

            // 011
            List<string> issns = GetFieldValues(root,
            nsmgr,
            "amazon:ItemAttributes/amazon:ISSN");

            // 200
            List<string> titles = GetFieldValues(root,
                nsmgr,
                "amazon:ItemAttributes/amazon:Title");
            List<Creator> creators = GetCreatorValues(root,
                nsmgr,
                "amazon:ItemAttributes/amazon:Creator");
            {
                MarcField field = new MarcField("200", "1 ");
                record.ChildNodes.add(field);

                foreach (string title in titles)
                {
                    field.ChildNodes.add(new MarcSubfield("a", title));
                }
                int i = 0;
                foreach (Creator creator in creators)
                {
                    field.ChildNodes.add(new MarcSubfield(i == 0 ? "f" : "g", creator.Name + creator.Role));
                    i++;
                }
            }

            // 2015/7/19
            // 205
            List<string> editions = GetFieldValues(root,
nsmgr,
"amazon:ItemAttributes/amazon:Edition");
            foreach (string edition in editions)
            {
                MarcField field = new MarcField("205", "  ");
                record.ChildNodes.add(field);

                field.ChildNodes.add(new MarcSubfield("a", edition));
            }

            // 210
            List<string> publishers = GetFieldValues(root,
    nsmgr,
    "amazon:ItemAttributes/amazon:Publisher");
            List<string> publication_dates = GetFieldValues(root,
nsmgr,
"amazon:ItemAttributes/amazon:PublicationDate");
            {
                MarcField field = new MarcField("210", "  ");
                record.ChildNodes.add(field);

                foreach (string s in publishers)
                {
                    field.ChildNodes.add(new MarcSubfield("c", s));
                }
                foreach (string s in publication_dates)
                {
                    // 日期字符串需要变换一下
                    field.ChildNodes.add(new MarcSubfield("d", GetPublishDateString(s)));
                }
            }

            // 215 a d
            List<string> pages = GetFieldValues(root,
nsmgr,
"amazon:ItemAttributes/amazon:NumberOfPages");
            List<string> heights = GetHeightValues(root,
nsmgr,
"amazon:ItemAttributes/amazon:PackageDimensions");
            {
                MarcField field = new MarcField("215", "  ");
                record.ChildNodes.add(field);

                foreach (string s in pages)
                {
                    field.ChildNodes.add(new MarcSubfield("a", s + "页"));
                }
                foreach (string s in heights)
                {
                    field.ChildNodes.add(new MarcSubfield("d", s));
                }
            }

            // 2015/7/19
            // 330
            List<string> reviews = GetCommentValues(root,
    nsmgr,
    "amazon:EditorialReviews/amazon:EditorialReview",
    "内容简介");
            if (reviews.Count > 0)
            {
                MarcField field = new MarcField("330", "  ");
                record.ChildNodes.add(field);
                foreach (string review in reviews)
                {
                    field.ChildNodes.add(new MarcSubfield("a", review));
                }
            }

            // 2015/7/19
            // 610
            List<string> subjects = GetFieldValues(root,
    nsmgr,
    "amazon:Subjects/amazon:Subject");
            if (subjects.Count > 0)
            {
                MarcField field = new MarcField("610", "0 ");
                record.ChildNodes.add(field);
                foreach (string subject in subjects)
                {
                    field.ChildNodes.add(new MarcSubfield("a", subject));
                }
            }

            // 2015/7/19
            // 7xx
            // authors 里面的元素通常已经被 creators 包含,creators 元素多余 authors。
            // 转换策略是,把 authors 里的作为 701,creators 里面余下的作为 702
            List<Creator> authors = GetCreatorValues(root,
    nsmgr,
    "amazon:ItemAttributes/amazon:Author");
            foreach (Creator author in authors)
            {
                MarcField field = new MarcField("701", " 0");
                record.ChildNodes.add(field);

                field.ChildNodes.add(new MarcSubfield("a", author.Name));
                if (string.IsNullOrEmpty(author.Role) == false)
                    field.ChildNodes.add(new MarcSubfield("4", author.Role));
            }

            foreach (Creator creator in creators)
            {
                if (IndexOf(authors, creator.Name) != -1)
                    continue;
                MarcField field = new MarcField("702", " 0");
                record.ChildNodes.add(field);

                field.ChildNodes.add(new MarcSubfield("a", creator.Name));
                if (string.IsNullOrEmpty(creator.Role) == false)
                    field.ChildNodes.add(new MarcSubfield("4", creator.Role));
            }

            if (StringUtil.IsInList("!856", strStyle) == false)
            {
                // 856
                string[] names = new string[] {
                "SmallImage",
                "MediumImage",
                "LargeImage"};
                foreach (string name in names)
                {
                    List<ImageInfo> small_images = GetImageValues(root,
        nsmgr,
        "amazon:" + name);
                    foreach (ImageInfo info in small_images)
                    {
                        MarcField field = new MarcField("856", "4 ");
                        record.ChildNodes.add(field);

                        field.ChildNodes.add(new MarcSubfield("3", "Cover image"));

                        field.ChildNodes.add(new MarcSubfield("u", info.Url));
                        field.ChildNodes.add(new MarcSubfield("q", GetMime(info.Url)));

                        field.ChildNodes.add(new MarcSubfield("x", "type:FrontCover." + name + ";size:" + info.Size + ";source:Amazon:"));
                        //  + dlg.SelectedItem.ASIN
                    }
                }
            }

            strMARC = record.Text;
            return 0;
        }

        static int IndexOf(List<Creator> list, string strName)
        {
            int i = 0;
            foreach (Creator creator in list)
            {
                if (creator.Name == strName)
                    return i;
                i++;
            }
            return -1;
        }

        // 根据 URL 获得媒体类型
        public static string GetMime(string strUrl)
        {
#if NO
            if (Path.GetExtension(strUrl).ToLower() == ".jpg")
                return "image/jpeg";
            return "";
#endif
            string strExt = Path.GetExtension(strUrl).ToLower();
            if (string.IsNullOrEmpty(strExt) == true)
                return "";

            return PathUtil.GetMimeTypeByFileExtension(strExt);
        }

        static string GetPublishDateString(string strText)
        {
            strText = strText.Replace("-", "");

            if (strText.Length < 6)
                return strText;

            return strText.Substring(0, 4) + "." + strText.Substring(4, 2).TrimStart(new char[] { '0' });
        }
    }

    [Serializable]  // 为了 Copy / Paste
    public class AmazonItemInfo
    {
        public string Xml = ""; // XML 记录
        public string EelementSet = ""; // "B" "F"
        public string PreferSyntaxOID = ""; // 优先要转换成的 MARC 格式
    }
}


text:
<?php

namespace App\Http\Controllers\Admin;

use Illuminate\Http\Request;

use App\Models\Origin;
use App\Http\Requests;
use App\Http\Controllers\Controller;

class OriginController extends Controller
{
    //

    public function index()
    {
        return view('admin.origins.index');
    }

    public function list() {
        $origins = Origin::orderby('created_at', 'desc')->get();

        return response()->json(['data' => $origins]);
    }

    public function show($id)
    {
    	$origin = Origin::find($id);
        return view('admin.origins.show', ['origin' => $origin]);
    }

    public function update(Request $request, $id)
    {
        $rules = [
            'contents' => 'required'
        ];

        $this->validate($request, $rules);

        $origin = Origin::find($id);
		$origin->save();

        return redirect()->route('admin.origin.show', ['id' => $id]);
    }
}

redpajama_set_name: RedPajamaGithub


text:
package hadoopInstaller.logging;

import hadoopInstaller.util.Messages;

import java.text.MessageFormat;

import org.apache.commons.logging.Log;

public class MessageFormattingLog {

	private Log log;

	public MessageFormattingLog(Log aLog) {
		this.log = aLog;
	}

	public void debug(String format, Object... arguments) {
		this.log.debug(MessageFormat.format(Messages.getString(format), arguments));
	}

	public void debug(Throwable t, String format, Object... arguments) {
		this.log.debug(MessageFormat.format(Messages.getString(format), arguments) + " - " + t.getLocalizedMessage());
		this.log.trace("trace", t);
	}

	public void error(String format, Object... arguments) {
		this.log.error(MessageFormat.format(Messages.getString(format), arguments));
	}

	public void error(Throwable t, String format, Object... arguments) {
		this.log.error(MessageFormat.format(Messages.getString(format), arguments) + " - " + t.getLocalizedMessage());
		this.log.trace("trace", t);
	}

	public void fatal(String format, Object... arguments) {
		this.log.fatal(MessageFormat.format(Messages.getString(format), arguments));
	}

	public void fatal(Throwable t, String format, Object... arguments) {
		this.log.fatal(MessageFormat.format(Messages.getString(format), arguments) + " - " + t.getLocalizedMessage());
		this.log.trace("trace", t);
	}

	public void info(String format, Object... arguments) {
		this.log.info(MessageFormat.format(Messages.getString(format), arguments));
	}

	public void info(Throwable t, String format, Object... arguments) {
		this.log.info(MessageFormat.format(Messages.getString(format), arguments) + " - " + t.getLocalizedMessage());
		this.log.trace("trace", t);
	}

	public boolean isDebugEnabled() {
		return this.log.isDebugEnabled();
	}

	public boolean isErrorEnabled() {
		return this.log.isErrorEnabled();
	}

	public boolean isFatalEnabled() {
		return this.log.isFatalEnabled();
	}

	public boolean isInfoEnabled() {
		return this.log.isInfoEnabled();
	}

	public boolean isTraceEnabled() {
		return this.log.isTraceEnabled();
	}

	public boolean isWarnEnabled() {
		return this.log.isWarnEnabled();
	}

	public void trace(String format, Object... arguments) {
		this.log.trace(MessageFormat.format(Messages.getString(format), arguments));
	}

	public void trace(Throwable t, String format, Object... arguments) {
		this.log.trace(MessageFormat.format(Messages.getString(format), arguments) + " - " + t.getLocalizedMessage(),
				t);
	}

	public void warn(String format, Object... arguments) {
		this.log.warn(MessageFormat.format(Messages.getString(format), arguments));
	}

	public void warn(Throwable t, String format, Object... arguments) {
		this.log.warn(MessageFormat.format(Messages.getString(format), arguments) + " - " + t.getLocalizedMessage());
		this.log.trace("trace", t);
	}
}


text:
namespace chainerx {
namespace cuda {

class CudnnError : public ChainerxError {
public:
    using ChainerxError::ChainerxError;

    explicit CudnnError(cudnnStatus_t status);
    cudnnStatus_t error() const noexcept { return status_; }

private:
    cudnnStatus_t status_{};
};

void CheckCudnnError(cudnnStatus_t status);

namespace cuda_internal {

// Returns a pointer to a cuDNN coefficient value of given type, allocated on the static storage.
template <int kValue>
const void* GetCudnnCoefficientPtr(Dtype dtype) {
    // TODO(niboshi): Get rid of the assumption that native and cuda float16 share the same representation.
    static const float kFloat32Value{kValue};
    static const double kFloat64Value{kValue};

    switch (dtype) {
        case Dtype::kFloat16:
            // fallthrough: cuDNN accepts float32 coefficients for float16 tensor operations.
        case Dtype::kFloat32:
            return &kFloat32Value;
        case Dtype::kFloat64:
            return &kFloat64Value;
        default:
            CHAINERX_NEVER_REACH();
    }
}

class CudnnTensorDescriptor {
public:
    CudnnTensorDescriptor();
    explicit CudnnTensorDescriptor(const Array& arr);

    ~CudnnTensorDescriptor();

    CudnnTensorDescriptor(const CudnnTensorDescriptor&) = delete;
    CudnnTensorDescriptor(CudnnTensorDescriptor&& other) noexcept : desc_{other.desc_} { other.desc_ = nullptr; }
    CudnnTensorDescriptor& operator=(const CudnnTensorDescriptor&) = delete;
    CudnnTensorDescriptor& operator=(CudnnTensorDescriptor&&) = delete;

    cudnnTensorDescriptor_t descriptor() const { return desc_; }
    cudnnTensorDescriptor_t operator*() const { return desc_; }

    Dtype GetDtype() const;

private:
    cudnnTensorDescriptor_t desc_{};
};

class CudnnFilterDescriptor {
public:
    explicit CudnnFilterDescriptor(const Array& w);

    ~CudnnFilterDescriptor();

    // TODO(hvy): Allow move semantics as needed.
    CudnnFilterDescriptor(const CudnnFilterDescriptor&) = delete;
    CudnnFilterDescriptor(CudnnFilterDescriptor&&) = delete;
    CudnnFilterDescriptor& operator=(const CudnnFilterDescriptor&) = delete;
    CudnnFilterDescriptor& operator=(CudnnFilterDescriptor&&) = delete;

    cudnnFilterDescriptor_t descriptor() const { return desc_; }
    cudnnFilterDescriptor_t operator*() const { return desc_; }

private:
    CudnnFilterDescriptor();
    cudnnFilterDescriptor_t desc_{};
};

class CudnnConvolutionDescriptor {
public:
    explicit CudnnConvolutionDescriptor(Dtype dtype, const Dims& pad, const Dims& stride, const absl::optional<Dims>& dilation, int groups);

    ~CudnnConvolutionDescriptor();

    // TODO(hvy): Allow move semantics as needed.
    CudnnConvolutionDescriptor(const CudnnConvolutionDescriptor&) = delete;
    CudnnConvolutionDescriptor(CudnnConvolutionDescriptor&&) = delete;
    CudnnConvolutionDescriptor& operator=(const CudnnConvolutionDescriptor&) = delete;
    CudnnConvolutionDescriptor& operator=(CudnnConvolutionDescriptor&&) = delete;

    cudnnConvolutionDescriptor_t descriptor() const { return desc_; }
    cudnnConvolutionDescriptor_t operator*() const { return desc_; }

private:
    CudnnConvolutionDescriptor();
    cudnnConvolutionDescriptor_t desc_{};
};

class CudnnPoolingDescriptor {
public:
    explicit CudnnPoolingDescriptor(
            cudnnPoolingMode_t mode,
            cudnnNanPropagation_t max_pooling_nan_opt,
            const Dims& kernel_size,
            const Dims& pad,
            const Dims& stride);

    ~CudnnPoolingDescriptor();

    // TODO(hvy): Allow move semantics as needed.
    CudnnPoolingDescriptor(const CudnnPoolingDescriptor&) = delete;
    CudnnPoolingDescriptor(CudnnPoolingDescriptor&&) = delete;
    CudnnPoolingDescriptor& operator=(const CudnnPoolingDescriptor&) = delete;
    CudnnPoolingDescriptor& operator=(CudnnPoolingDescriptor&&) = delete;

    cudnnPoolingDescriptor_t descriptor() const { return desc_; }
    cudnnPoolingDescriptor_t operator*() const { return desc_; }

private:
    CudnnPoolingDescriptor();
    cudnnPoolingDescriptor_t desc_{};
};

// cuDNN API calls using same handle is not thread-safe.
// This class ensures that the API calls are serialized using mutex lock.
class CudnnHandle {
public:
    explicit CudnnHandle(int device_index) : device_index_{device_index} {}
    ~CudnnHandle();

    CudnnHandle(const CudnnHandle&) = delete;
    CudnnHandle(CudnnHandle&&) = delete;
    CudnnHandle& operator=(const CudnnHandle&) = delete;
    CudnnHandle& operator=(CudnnHandle&&) = delete;

    template <class Func, class... Args>
    void Call(Func&& func, Args&&... args) {
        std::lock_guard<std::mutex> lock{handle_mutex_};
        CheckCudnnError(func(handle(), args...));
    }

private:
    cudnnHandle_t handle();

    int device_index_;
    std::mutex handle_mutex_{};
    cudnnHandle_t handle_{};
};

}  // namespace cuda_internal
}  // namespace cuda
}  // namespace chainerx


text:
#if !defined(BOOST_PP_IS_ITERATING)

#ifndef BOOST_TYPE_ERASURE_DETAIL_ADAPT_TO_VTABLE_HPP_INCLUDED
#define BOOST_TYPE_ERASURE_DETAIL_ADAPT_TO_VTABLE_HPP_INCLUDED

#include <boost/utility/addressof.hpp>
#include <boost/mpl/if.hpp>
#include <boost/mpl/eval_if.hpp>
#include <boost/mpl/has_xxx.hpp>
#include <boost/type_traits/function_traits.hpp>
#include <boost/type_traits/remove_cv.hpp>
#include <boost/preprocessor/cat.hpp>
#include <boost/preprocessor/iteration/iterate.hpp>
#include <boost/preprocessor/repetition/enum.hpp>
#include <boost/preprocessor/repetition/enum_params.hpp>
#include <boost/preprocessor/repetition/enum_trailing_params.hpp>
#include <boost/preprocessor/repetition/enum_binary_params.hpp>
#include <boost/type_erasure/detail/get_signature.hpp>
#include <boost/type_erasure/detail/storage.hpp>
#include <boost/type_erasure/is_placeholder.hpp>
#include <boost/type_erasure/config.hpp>

namespace boost {
namespace type_erasure {

namespace detail {

template<class T, class Bindings>
struct rebind_placeholders;

template<class T, class Bindings>
struct rebind_placeholders_in_argument;

template<class PrimitiveConcept, class Sig>
struct vtable_adapter;

template<class PrimitiveConcept, class Sig, class Bindings>
struct rebind_placeholders<vtable_adapter<PrimitiveConcept, Sig>, Bindings>
{
    typedef vtable_adapter<
        typename rebind_placeholders<PrimitiveConcept, Bindings>::type,
        typename rebind_placeholders_in_argument<Sig, Bindings>::type
    > type;
};

template<class T>
struct replace_param_for_vtable
{
    typedef typename ::boost::mpl::if_<
        ::boost::type_erasure::is_placeholder<typename ::boost::remove_cv<T>::type>,
        const ::boost::type_erasure::detail::storage&,
        T
    >::type type;
};

template<class T>
struct replace_param_for_vtable<T&>
{
    typedef typename ::boost::mpl::if_<
        ::boost::type_erasure::is_placeholder<typename ::boost::remove_cv<T>::type>,
        ::boost::type_erasure::detail::storage&,
        T&
    >::type type;
};

template<class T>
struct replace_param_for_vtable<const T&>
{
    typedef typename ::boost::mpl::if_<
        ::boost::type_erasure::is_placeholder<typename ::boost::remove_cv<T>::type>,
        const ::boost::type_erasure::detail::storage&,
        const T&
    >::type type;
};

#ifndef BOOST_NO_CXX11_RVALUE_REFERENCES

template<class T>
struct replace_param_for_vtable<T&&>
{
    typedef typename ::boost::mpl::if_<
        ::boost::type_erasure::is_placeholder<typename ::boost::remove_cv<T>::type>,
        ::boost::type_erasure::detail::storage&&,
        T&&
    >::type type;
};

#endif

template<class T>
struct replace_result_for_vtable
{
    typedef typename ::boost::mpl::if_<
        ::boost::type_erasure::is_placeholder<typename ::boost::remove_cv<T>::type>,
        ::boost::type_erasure::detail::storage,
        T
    >::type type;
};

template<class T>
struct replace_result_for_vtable<T&>
{
    typedef typename ::boost::mpl::if_<
        ::boost::type_erasure::is_placeholder<typename ::boost::remove_cv<T>::type>,
        ::boost::type_erasure::detail::storage&,
        T&
    >::type type;
};

template<class T>
struct replace_result_for_vtable<const T&>
{
    typedef typename ::boost::mpl::if_<
        ::boost::type_erasure::is_placeholder<typename ::boost::remove_cv<T>::type>,
        ::boost::type_erasure::detail::storage&,
        const T&
    >::type type;
};

#ifndef BOOST_NO_CXX11_RVALUE_REFERENCES

template<class T>
struct replace_result_for_vtable<T&&>
{
    typedef typename ::boost::mpl::if_<
        ::boost::type_erasure::is_placeholder<typename ::boost::remove_cv<T>::type>,
        ::boost::type_erasure::detail::storage&&,
        T&&
    >::type type;
};

#endif

template<class Sig>
struct get_vtable_signature;

BOOST_MPL_HAS_XXX_TRAIT_DEF(type)

template<class T>
struct is_internal_concept :
    ::boost::type_erasure::detail::has_type<T>
{};

template<class PrimitiveConcept>
struct adapt_to_vtable
{
    typedef ::boost::type_erasure::detail::vtable_adapter<
        PrimitiveConcept,
        typename ::boost::type_erasure::detail::get_vtable_signature<
            typename ::boost::type_erasure::detail::get_signature<
                PrimitiveConcept
            >::type
        >::type
    > type;
};

template<class Concept>
struct maybe_adapt_to_vtable
{
    typedef typename ::boost::mpl::eval_if<
        ::boost::type_erasure::detail::is_internal_concept<Concept>,
        ::boost::mpl::identity<Concept>,
        ::boost::type_erasure::detail::adapt_to_vtable<Concept>
    >::type type;
};

#if !defined(BOOST_NO_CXX11_VARIADIC_TEMPLATES) && !defined(BOOST_NO_CXX11_RVALUE_REFERENCES)

template<class PrimitiveConcept, class Sig, class ConceptSig>
struct vtable_adapter_impl;

template<class PrimitiveConcept, class R, class... T, class R2, class... U>
struct vtable_adapter_impl<PrimitiveConcept, R(T...), R2(U...)>
{
    typedef R (*type)(T...);
    static R value(T... arg)
    {
        return PrimitiveConcept::apply(
            ::boost::type_erasure::detail::extract<U>(std::forward<T>(arg))...);
    }
};

template<class PrimitiveConcept, class... T, class R2, class... U>
struct vtable_adapter_impl<PrimitiveConcept, ::boost::type_erasure::detail::storage&(T...), R2(U...)>
{
    typedef ::boost::type_erasure::detail::storage (*type)(T...);
    static ::boost::type_erasure::detail::storage value(T... arg)
    {
        ::boost::type_erasure::detail::storage result;
        typename ::boost::remove_reference<R2>::type* p =
            ::boost::addressof(
                PrimitiveConcept::apply(::boost::type_erasure::detail::extract<U>(std::forward<T>(arg))...));
        result.data = const_cast<void*>(static_cast<const void*>(p));
        return result;
    }
};

template<class PrimitiveConcept, class... T, class R2, class... U>
struct vtable_adapter_impl<PrimitiveConcept, ::boost::type_erasure::detail::storage&&(T...), R2(U...)>
{
    typedef ::boost::type_erasure::detail::storage (*type)(T...);
    static ::boost::type_erasure::detail::storage value(T... arg)
    {
        ::boost::type_erasure::detail::storage result;
        R2 tmp = PrimitiveConcept::apply(::boost::type_erasure::detail::extract<U>(std::forward<T>(arg))...);
        typename ::boost::remove_reference<R2>::type* p = ::boost::addressof(tmp);
        result.data = const_cast<void*>(static_cast<const void*>(p));
        return result;
    }
};

template<class PrimitiveConcept, class Sig>
struct vtable_adapter
    : vtable_adapter_impl<
        PrimitiveConcept,
        Sig,
        typename ::boost::type_erasure::detail::get_signature<
            PrimitiveConcept
        >::type
    >
{};

template<class R, class... T>
struct get_vtable_signature<R(T...)>
{
    typedef typename ::boost::type_erasure::detail::replace_result_for_vtable<
        R
    >::type type(typename ::boost::type_erasure::detail::replace_param_for_vtable<T>::type...);
};

#else

#define BOOST_PP_FILENAME_1 <boost/type_erasure/detail/adapt_to_vtable.hpp>
#define BOOST_PP_ITERATION_LIMITS (0, BOOST_TYPE_ERASURE_MAX_ARITY)
#include BOOST_PP_ITERATE()

#endif

}
}
}

#endif

#else

#define N BOOST_PP_ITERATION()

#ifndef BOOST_NO_CXX11_RVALUE_REFERENCES

#define BOOST_TYPE_ERASURE_EXTRACT(z, n, data)                  \
    ::boost::type_erasure::detail::extract<                     \
        typename traits::                                       \
        BOOST_PP_CAT(BOOST_PP_CAT(arg, BOOST_PP_INC(n)), _type) \
    >(std::forward<BOOST_PP_CAT(T, n)>(BOOST_PP_CAT(arg, n)))

#else

#define BOOST_TYPE_ERASURE_EXTRACT(z, n, data)                  \
    ::boost::type_erasure::detail::extract<                     \
        typename traits::                                       \
        BOOST_PP_CAT(BOOST_PP_CAT(arg, BOOST_PP_INC(n)), _type) \
    >(BOOST_PP_CAT(arg, n))

#endif

#define BOOST_TYPE_ERASURE_REPLACE_PARAM(z, n, data)                    \
    typename ::boost::type_erasure::detail::replace_param_for_vtable<   \
        BOOST_PP_CAT(T, n)>::type

template<class PrimitiveConcept, class R
    BOOST_PP_ENUM_TRAILING_PARAMS(N, class T)>
struct vtable_adapter<PrimitiveConcept, R(BOOST_PP_ENUM_PARAMS(N, T))>
{
    typedef R (*type)(BOOST_PP_ENUM_PARAMS(N, T));
    static R value(BOOST_PP_ENUM_BINARY_PARAMS(N, T, arg))
    {
        typedef typename ::boost::function_traits<
            typename ::boost::type_erasure::detail::get_signature<
                PrimitiveConcept
            >::type
        > traits;
        return PrimitiveConcept::apply(
            BOOST_PP_ENUM(N, BOOST_TYPE_ERASURE_EXTRACT, ~));
    }
};

template<class PrimitiveConcept
    BOOST_PP_ENUM_TRAILING_PARAMS(N, class T)>
struct vtable_adapter<PrimitiveConcept, ::boost::type_erasure::detail::storage&(BOOST_PP_ENUM_PARAMS(N, T))>
{
    typedef ::boost::type_erasure::detail::storage (*type)(BOOST_PP_ENUM_PARAMS(N, T));
    static ::boost::type_erasure::detail::storage value(BOOST_PP_ENUM_BINARY_PARAMS(N, T, arg))
    {
        typedef typename ::boost::function_traits<
            typename ::boost::type_erasure::detail::get_signature<
                PrimitiveConcept
            >::type
        > traits;
        ::boost::type_erasure::detail::storage result;
        typename ::boost::remove_reference<typename traits::result_type>::type* p =
            ::boost::addressof(
                PrimitiveConcept::apply(BOOST_PP_ENUM(N, BOOST_TYPE_ERASURE_EXTRACT, ~)));
        result.data = const_cast<void*>(static_cast<const void*>(p));
        return result;
    }
};

#ifndef BOOST_NO_CXX11_RVALUE_REFERENCES

template<class PrimitiveConcept
    BOOST_PP_ENUM_TRAILING_PARAMS(N, class T)>
struct vtable_adapter<PrimitiveConcept, ::boost::type_erasure::detail::storage&&(BOOST_PP_ENUM_PARAMS(N, T))>
{
    typedef ::boost::type_erasure::detail::storage (*type)(BOOST_PP_ENUM_PARAMS(N, T));
    static ::boost::type_erasure::detail::storage value(BOOST_PP_ENUM_BINARY_PARAMS(N, T, arg))
    {
        typedef typename ::boost::function_traits<
            typename ::boost::type_erasure::detail::get_signature<
                PrimitiveConcept
            >::type
        > traits;
        ::boost::type_erasure::detail::storage result;
        typename traits::result_type tmp =
            PrimitiveConcept::apply(BOOST_PP_ENUM(N, BOOST_TYPE_ERASURE_EXTRACT, ~));
        typename ::boost::remove_reference<typename traits::result_type>::type* p =
            ::boost::addressof(tmp);
        result.data = const_cast<void*>(static_cast<const void*>(p));
        return result;
    }
};

#endif

template<class R BOOST_PP_ENUM_TRAILING_PARAMS(N, class T)>
struct get_vtable_signature<R(BOOST_PP_ENUM_PARAMS(N, T))>
{
    typedef typename ::boost::type_erasure::detail::replace_result_for_vtable<
        R
    >::type type(BOOST_PP_ENUM(N, BOOST_TYPE_ERASURE_REPLACE_PARAM, ~));
};

#undef BOOST_TYPE_ERASURE_REPLACE_PARAM
#undef BOOST_TYPE_ERASURE_EXTRACT
#undef N

#endif


text:
/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.qcert.util;

import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.qcert.util.SchemaUtil.ListType;
import org.qcert.util.SchemaUtil.ObjectType;
import org.qcert.util.SchemaUtil.PrimitiveType;
import org.qcert.util.SchemaUtil.Type;

import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
import com.google.gson.internal.Streams;
import com.google.gson.stream.JsonWriter;

/**
 * Generates the standard qcert input format from one or more "CSV" files and a qcert schema.
 *
 *
There are two public methods, a loadData method, which is flexible enough to handle all sorts of file naming conventions
 *   and "CSV" formats, and a main method, which can be invoked from the command line but requires a strict file naming
 *   convention and handles only strict the RFC 4180 CSV interpretation, with a required header line in each file.
 *
 *
Most generally, the input consists of
 *

a set of CSV files and their associated internal type ("table") names.  On the command line, the association is strict:
 *   the file name must be the table name plus the suffix .csv and be found in a specific directory.  For loadData
 *   the association is given explicitly in a Map argument.
 *
a CSV format designation (all formats supported by Apache commons-csv, but a header must be available somehow,
 *  either from the format or from each file).  On the command line, the format is not specified and must be RFC 4180 with in-file
 *  headers (the delimiter may optionally be set to something other than comma).
 *
a qcert schema.  On the command line, this is given as a file path.  For loadData it is given as a JsonElement.
 *

 *
Each CSV file should contain just data for its corresponding internal type.  That type may be either a brand in
 *   the schema's brandTypes or a partition name in the schema's globals, depending on the memory
 *  layout intended.
 *
The schema should be in one of qcert standard JSON formats supported by SchemaUtil.
 *   The two different ways that the set of "table" names can match names in the schema gives rise to a different memory layout.
 *

If the table name matches a brand in the brandTypes, the type definition is taken from typeDefs
 *  and the input of that type will go in the partition called "WORLD", marked by its type.
 *  This is a heuristic but works for present cases because, depending on the source language, we either assume this case or the
 *   next one.
 *
If the table name matches a member of globals, the type definition is the type of that member
 *  and input of that type will go in a homogeneous memory partition named for the type.
 *

 * In either case, the names of the attributes in the type definition are matched against header names in the CSV file.  A column in
 * the CSV file that does not match an attribute name is an error.
 *
For this to work, all types in the schema must be primitive (numbers, booleans strings, and dates).  Embedded list and
 * object types are not handled.
 *
Currently, if the type is "date" (ignoring case), the field is formatted as a date JSON object with members "year", "month" and "day".
 * This is a good convention for SQL but other formats probably need to be investigated.
 */
public class DataLoader {
	private DataLoader() {}

	/** Primary entry point to this service.  The 'main' method is also useful but can be bypassed to call this instead
	 *   when files are named in non-standard ways or to use a CSV format understood by Apache commons-csv but other than strict RFC 4180.
	 * @param tableMap a map from "table names" (type names matching CSV files) to the corresponding CSV file contents (as Strings)
	 * @param jsonSchema the JSON format schema in one of several possible variations, as a JsonElement (array or object)
	 * @param format the CSV format to use (if the format does not include a header we assume that the first line of
	 *   each file constitutes a header
	 * @return a JsonObject representing the loaded data
	 * @throws Exception
	 */
	public static JsonObject loadData(Map<String, String> tableMap, JsonElement jsonSchema, CSVFormat format) throws Exception {
		if (format.getHeader() == null)
			format = format.withHeader();
		Map<String, ObjectType> types = SchemaUtil.getSchema(jsonSchema);
		boolean partitioned = types.size() == 0;
		if (partitioned) {
			if (jsonSchema.isJsonObject())
				types = SchemaUtil.getGlobalsFromSchema(jsonSchema.getAsJsonObject());
			if (types.size() == 0)
				throw new IllegalArgumentException("Schema contains no useful information");
		}

		/* Process the data files */
		JsonObject ans = new JsonObject();
		JsonArray world = partitioned ? null : new JsonArray();
		if (!partitioned)
			ans.add("WORLD", world);
		for (Entry<String, String> filePair : tableMap.entrySet()) {
			String table = filePair.getKey();
			ObjectType def = types.get(table);
			if (def == null)
				throw new IllegalArgumentException("Type " + table + " is not in the schema");
			JsonArray thisType = process(filePair.getValue(), def, format);
			if (partitioned)
				ans.add(table, thisType);
			else {
				for (JsonElement elem : thisType) {
					JsonObject toAdd = new JsonObject();
					JsonArray brands = new JsonArray();
					brands.add(new JsonPrimitive(table));
					toAdd.add("type", brands);
					toAdd.add("data", elem);
					world.add(toAdd);
				}
			}
		}
		return ans;
	}

	/**
	 * Main program.
	 *
Command line arguments are
	 *

-dir <path> (optional) the directory in which to find or create all other files, defaults to current directory
	 *
-output <filename> (required) the name of the output file (absolute or relative to -dir)
	 *
-schema <filename> (required) the name of the schema file (absolute or relative to -dir)
	 *
-delimiter <char≷ (optional) a character to use as delimiter (defaults to comma; if the chose character is shell-sensitive, be sure to quote it)
	 *
all other arguments are assumed to be type names.  A file of that name with extension .csv must be present in -dir
	 *

	 */
	public static void main(String[] args) throws Exception {

		/* Parse the command line */
		String directory = null, schema = null, output = null; char delimiter = 0;
		List<String> tables = new ArrayList<>();
		boolean dirFlag = false, outFlag = false, schemaFlag = false, delimiterFlag = false;
		for (String arg : args) {
			boolean table = false;
			if (dirFlag)
				directory = arg;
			else if (outFlag)
				output = arg;
			else if (schemaFlag)
				schema = arg;
			else if (delimiterFlag) {
				if (arg.length() != 1)
					throw new IllegalArgumentException("Delimiters must be single character");
				else
					delimiter = arg.charAt(0);
			} else
				table = true;
			dirFlag = outFlag = schemaFlag = delimiterFlag = false;
			if (arg.charAt(0) == '-') {
				switch(arg) {
				case "-dir":
					if (directory != null)
						throw new IllegalArgumentException("Duplicate -dir");
					else
						dirFlag = true;
					continue;
				case "-output":
					if (output != null)
						throw new IllegalArgumentException("Duplicate -output");
					else
						outFlag = true;
					continue;
				case "-schema":
					if (schema != null)
						throw new IllegalArgumentException("Duplicate -schema");
					else
						schemaFlag = true;
					continue;
				case "-delimiter":
					if (delimiter != 0)
						throw new IllegalArgumentException("Duplicate -delimiter");
					else
						delimiterFlag = true;
					continue;
				default:
					throw new IllegalArgumentException("Unknown option :" + arg);
				}
			}
			if (table)
				tables.add(arg);
		}
		if (output == null)
			throw new IllegalArgumentException("Output file must be specified");
		if (schema == null)
			throw new IllegalArgumentException("Schema file must be specified");
		File outputFile = new File(output);
		File schemaFile = new File(schema);
		if (directory != null) {
			if (!outputFile.isAbsolute())
				outputFile = new File(directory, output);
			if (!schemaFile.isAbsolute())
				schemaFile = new File(directory, schema);
		}

		/* Canonicalize the file names, read the contents, and associate each with its corresponding table name */
		Map<String, String> tableMap = new HashMap<>();
		for (String table : tables) {
			File toRead = directory == null ? new File(table + ".csv") : new File(directory, table + ".csv");
			String contents = new String(Files.readAllBytes(Paths.get(toRead.getAbsolutePath())));
			tableMap.put(table, contents);
		}

		/* Parse the schema */
		JsonElement jsonSchema = new JsonParser().parse(new FileReader(schemaFile));

		/* Process, using callable service */
		CSVFormat format = CSVFormat.RFC4180;
		if (delimiter != 0)
			format = format.withDelimiter(delimiter);
		JsonObject ans = loadData(tableMap, jsonSchema, format);

		/* Write the result */
		FileWriter wtr = new FileWriter(outputFile);
		JsonWriter json = new JsonWriter(wtr);
		json.setLenient(true);
		json.setIndent("  ");
		Streams.write(ans, json);
		json.close();
	}

	/**
	 * Convert a primitive value of designated type to a JsonElement
	 * @param value the value to convert
	 * @param fieldType the type of the field
	 * @return a JsonElement (either a primitive or a date object)
	 */
	private static JsonElement convertPrimitiveValue(String value, Type fieldType) {
		String typeName = ((PrimitiveType) fieldType).typeName;
		switch(typeName) {
		case "String":
			return new JsonPrimitive(value);
		case "Nat":
      JsonObject ans = new JsonObject();
			/* We are a little loosy-goosy with numbers since the sources of information are often inexact */
			try {
			    ans.add("$nat",new JsonPrimitive(Integer.parseInt(value)));
			    return ans;
			} catch (NumberFormatException ig) {
			    ans.add("$nat",new JsonPrimitive(Double.parseDouble(value)));
			    return ans;
			}
		case "Float":
        return new JsonPrimitive(Double.parseDouble(value));
		case "ESqlDate":
			return formatDate(value);
		case "Bool":
			return new JsonPrimitive(value.equalsIgnoreCase("true"));
		default:
			throw new UnsupportedOperationException("Don't known how to convert primitive schema type " + typeName);
		}
	}

	/**
	 * Format a SQL-style String date into a JSON date object, which, by convention, we use for this type
	 * TODO support other kinds of dates
	 * @param stringDate the date to format
	 * @return the JSON result
	 */
	private static JsonElement formatDate(String stringDate) {
		String[] dateParts = stringDate.split("-");
		assert dateParts.length == 3;
		String[] names = {"year",  "month", "day"};
		JsonObject ans = new JsonObject();
		JsonObject ansIn = new JsonObject();
		JsonObject ansIn2 = new JsonObject();
		for (int i = 0; i < 3; i++) {
      ansIn2.add(names[i], new JsonPrimitive(Integer.parseInt(dateParts[i])));
		}
    ansIn.add("$date", ansIn2);
    ans.add("$foreign", ansIn);
		return ans;
	}

	/** Process an individual table, producing its rows in JSON form
	 * @param data the CSV file contents as a String
	 * @param def the type definition as an ObjectType
	 * @param format the CSVFormat to use
	 * @return a JsonArray of the translation of the rows
	 * @throws Exception
	 */
	private static JsonArray process(String data, ObjectType def, CSVFormat format) throws Exception {
		JsonArray ans = new JsonArray();
		List<CSVRecord> records = CSVParser.parse(data, format).getRecords();
		for (CSVRecord record : records) {
			Map<String, String> recmap = record.toMap();
			JsonObject datum = new JsonObject();
			for (Entry<String, String> col : recmap.entrySet()) {
				datum.add(col.getKey(), processColumn(col.getKey(), col.getValue(), def));
			}
			ans.add(datum);
		}
		return ans;
	}

	/**
	 * Process an individual column of an individual row of an individual table, producing its JSON representation as a JSON object
	 * @param fieldName the name of the column
	 * @param value the raw (String) value in the CSV file
	 * @param def the ObjectType that should produce the type for each field name, allowing the value to be interpreted
	 * @return a JsonObject encoding the column of the row
	 */
	private static JsonElement processColumn(String fieldName, String value, ObjectType def) {
		if (fieldName == null)
			throw new IllegalArgumentException("No header information in CSV file");
		Type fieldType = def.attributes.get(fieldName);
		if (fieldType == null)
			throw new IllegalArgumentException("Field " + fieldName + " is not among the attributes of type " + def.brand);
		if (fieldType instanceof PrimitiveType) {
			return convertPrimitiveValue(value, fieldType);
		}
		if (fieldType instanceof ListType) {
			Type elementType = ((ListType) fieldType).elementType;
			if (elementType instanceof PrimitiveType) {
				JsonArray ans = new JsonArray();
				String[] values = value.split(",");
				for (String val : values)
					ans.add(convertPrimitiveValue(val, elementType));
				return ans;
			}
		}
		throw new UnsupportedOperationException("Can't handle embedded object types, lists thereof, or lists of lists, when loading data");
	}
}


text:
package com.sequenceiq.cloudbreak.api.model;

public enum SssdSchemaType {

    RFC2307("rfc2307"),
    RFC2307BIS("rfc2307bis"),
    IPA("IPA"),
    AD("AD");

    private String representation;

    SssdSchemaType(String representation) {
        this.representation = representation;
    }

    public String getRepresentation() {
        return representation;
    }
}


text:
/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.cekrlic.jlesscss;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.servlet.ServletOutputStream;
import javax.servlet.WriteListener;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpServletResponseWrapper;
import java.io.*;

/**
 * Response wrapper to capture resources.
 */
public class ResourceResponseWrapper extends HttpServletResponseWrapper {
	private static final Logger log = LoggerFactory.getLogger(ResourceResponseWrapper.class);

	final Object guard = new Object();
	final ByteArrayOutputStream bos = new ByteArrayOutputStream();

	PrintWriter pw = null;
	ServletOutputStream os = new PassThroughServletOutputStream(bos);

	static class PassThroughServletOutputStream extends ServletOutputStream {
		final OutputStream bos;

		PassThroughServletOutputStream(final OutputStream os) {
			this.bos = os;
		}

		public void write(int b) throws IOException {
			bos.write(b);
		}

		public void write(byte[] b) throws IOException {
			bos.write(b);
		}

		public void write(byte[] b, int off, int len) throws IOException {
			bos.write(b, off, len);
		}

		@Override
		public boolean isReady() {
			return true;
		}

		@Override
		public void setWriteListener(WriteListener writeListener) {
			try {
				writeListener.onWritePossible();
			} catch (IOException e) {
				throw new RuntimeException(e);
			}
		}
	}

	/**
	 * Constructor, set the original response.
	 *
	 * @param response
	 */
	public ResourceResponseWrapper(HttpServletResponse response) {
		super(response);
	}

	@Override
	public void setContentType(String type) {
		super.setContentType(type);
		log.debug("Setting content type to: {}", type);
	}

	@Override
	public ServletOutputStream getOutputStream() throws IOException {
		return os;
	}

	@Override
	public PrintWriter getWriter() throws IOException {
		synchronized (guard) {
			if (pw == null) {
				OutputStreamWriter osw = new OutputStreamWriter(os, getCharacterEncoding());
				pw = new PrintWriter(osw, true);
			}
		}
		return pw;
	}

	public byte[] toByteArray() {
		try {
			synchronized (guard) {
				if (pw != null) {
					pw.flush();
				}
				os.flush();
			}
			return bos.toByteArray();
		} catch (IOException e) {
			throw new RuntimeException("Could not flush stream: " + e.toString(), e);
		}
	}

	public String toString() {
		try {
			return new String(toByteArray(), getCharacterEncoding());
		} catch (UnsupportedEncodingException e) {
			throw new RuntimeException("Encoding not supported: " + e.toString(), e);
		}
	}

}


text:
class Page < ActiveRecord::Base
  validates :title, :presence => true

  acts_as_nested_set

  # Docs for friendly_id http://github.com/norman/friendly_id
  has_friendly_id :title, :use_slug => true,
                  :reserved_words => %w(index new session login logout users refinery admin images wymiframe)

  has_many :parts,
           :class_name => "PagePart",
           :order => "position ASC",
           :inverse_of => :page,
           :dependent => :destroy

  accepts_nested_attributes_for :parts,
                                :allow_destroy => true

  # Docs for acts_as_indexed http://github.com/dougal/acts_as_indexed
  acts_as_indexed :fields => [:title, :meta_keywords, :meta_description,
                              :custom_title, :browser_title, :all_page_part_content]

  before_destroy :deletable?
  after_save :reposition_parts!
  after_save :invalidate_child_cached_url

  # when a dialog pops up to link to a page, how many pages per page should there be
  PAGES_PER_DIALOG = 14

  # when listing pages out in the admin area, how many pages should show per page
  PAGES_PER_ADMIN_INDEX = 20

  # when collecting the pages path how is each of the pages seperated?
  PATH_SEPARATOR = " - "

  # Am I allowed to delete this page?
  # If a link_url is set we don't want to break the link so we don't allow them to delete
  # If deletable is set to false then we don't allow this page to be deleted. These are often Refinery system pages
  def deletable?
    self.deletable && self.link_url.blank? and self.menu_match.blank?
  end

  # Repositions the child page_parts that belong to this page.
  # This ensures that they are in the correct 0,1,2,3,4... etc order.
  def reposition_parts!
    self.parts.each_with_index do |part, index|
      part.update_attribute(:position, index)
    end
  end

  # Before destroying a page we check to see if it's a deletable page or not
  # Refinery system pages are not deletable.
  def destroy
    if self.deletable?
      super
    else
      unless Rails.env.test?
        # give useful feedback when trying to delete from console
        puts "This page is not deletable. Please use .destroy! if you really want it deleted "
        puts "unset .link_url," if self.link_url.present?
        puts "unset .menu_match," if self.menu_match.present?
        puts "set .deletable to true" unless self.deletable
      end

      return false
    end
  end

  # If you want to destroy a page that is set to be not deletable this is the way to do it.
  def destroy!
    self.menu_match = nil
    self.link_url = nil
    self.deletable = true

    self.destroy
  end

  def indented_title
    "#{"--" * self.ancestors.size} #{self.title}".chomp
  end

  # Used for the browser title to get the full path to this page
  # It automatically prints out this page title and all of it's parent page titles joined by a PATH_SEPARATOR
  def path(reverse = true)
    unless self.parent.nil?
      parts = [self.title, self.parent.path(reverse)]
      parts.reverse! if reverse
      parts.join(PATH_SEPARATOR)
    else
      self.title
    end
  end

  # When this page is rendered in the navigation, where should it link?
  # If a custom "link_url" is set, it uses that otherwise it defaults to a normal page URL.
  # The "link_url" is often used to link to a plugin rather than a page.
  #
  # For example if I had a "Contact" page I don't want it to just render a contact us page
  # I want it to show the Inquiries form so I can collect inquiries. So I would set the "link_url"
  # to "/contact"
  def url
    if self.link_url.present?
      link_url_localised?
    elsif use_marketable_urls?
      url_marketable
    elsif self.to_param.present?
      url_normal
    end
  end

  def link_url_localised?
    if self.link_url =~ /^\// and defined?(::Refinery::I18n) and ::Refinery::I18n.enabled? and
       ::I18n.locale != ::Refinery::I18n.default_frontend_locale
      "/#{::I18n.locale}#{self.link_url}"
    else
      self.link_url
    end
  end

  def url_marketable
    # :id => nil is important to prevent any other params[:id] from interfering with this route.
    {:controller => "/pages", :action => "show", :path => self.nested_url, :id => nil}
  end

  def url_normal
    {:controller => "/pages", :action => "show", :id => self.to_param}
  end

  # Returns an array with all ancestors to_param, allow with its own
  # Ex: with an About page and a Mission underneath,
  # Page.find('mission').nested_url would return:
  #
  #   ['about', 'mission']
  #
  def nested_url
    Rails.cache.fetch(url_cache_key) { uncached_nested_url }
  end

  def uncached_nested_url
    self.parent ? [parent.nested_url, self.to_param].flatten : [self.to_param]
  end

  # Returns the string version of nested_url, i.e., the path that should be generated
  # by the router
  def nested_path
    @nested_path ||= "/#{nested_url.join('/')}"
  end

  def url_cache_key
    "#{cache_key}#nested_url"
  end

  def cache_key
    "#{Refinery.base_cache_key}/#{super}"
  end

  def use_marketable_urls?
    RefinerySetting.find_or_set(:use_marketable_urls, true, :scoping => 'pages')
  end

  # Returns true if this page is "published"
  def live?
    not self.draft?
  end

  # Return true if this page can be shown in the navigation.
  # If it's a draft or is set to not show in the menu it will return false.
  # If any of the page's ancestors aren't to be shown in the menu then this page is not either.
  def in_menu?
    self.live? && self.show_in_menu? && self.ancestors.all? { |a| a.in_menu? }
  end

  # Returns true if this page is the home page or links to it.
  def home?
    self.link_url == "/"
  end

  # Returns all visible sibling pages that can be rendered for the menu
  def shown_siblings
    self.siblings.reject { |sibling| not sibling.in_menu? }
  end

  class << self
    # Accessor to find out the default page parts created for each new page
    def default_parts
      RefinerySetting.find_or_set(:default_page_parts, ["Body", "Side Body"])
    end

    # Returns how many pages per page should there be when paginating pages
    def per_page(dialog = false)
      dialog ? PAGES_PER_DIALOG : PAGES_PER_ADMIN_INDEX
    end

    # Returns all the top level pages, usually to render the top level navigation.
    def top_level
      self.roots.where(:show_in_menu => true, :draft => false)
    end
  end

  # Accessor method to get a page part from a page.
  # Example:
  #
  #    Page.first[:body]
  #
  # Will return the body page part of the first page.
  def [](part_title)
    # don't want to override a super method when trying to call a page part.
    # the way that we call page parts seems flawed, will probably revert to page.parts[:title] in a future release.
    if (super_value = super).blank?
      # self.parts is already eager loaded so we can now just grab the first element matching the title we specified.
      part = self.parts.detect do |part|
        part.title == part_title.to_s or
        part.title.downcase.gsub(" ", "_") == part_title.to_s.downcase.gsub(" ", "_")
      end

      return part.body unless part.nil?
    end

    super_value
  end

  # In the admin area we use a slightly different title to inform the which pages are draft or hidden pages
  def title_with_meta
    title = self.title
    title << " (hidden)" unless self.show_in_menu?
    title << " (draft)" if self.draft?

    title.strip
  end

  # Used to index all the content on this page so it can be easily searched.
  def all_page_part_content
    self.parts.collect {|p| p.body}.join(" ")
  end

  ##
  # Protects generated slugs from title if they are in the list of reserved words
  # This applies mostly to plugin-generated pages.
  #
  # Returns the sluggified string
  def normalize_friendly_id(slug_string)
    sluggified = super
    if use_marketable_urls? && self.class.friendly_id_config.reserved_words.include?(sluggified)
      sluggified += "-page"
    end
    sluggified
  end

  private

  def invalidate_child_cached_url
    return true unless use_marketable_urls?
    children.each do |child|
      Rails.cache.delete(child.url_cache_key)
    end
  end
end