<file_sep>/**
 * Created by jeanzhao on 10/23/18.
 */

export const DefaultConfig = {

    // puppeteer launch setting
    launchOptions: {
        headless: false,
        defaultViewport: {
            width: 1024,
            height: 768
        },
        timeout: 60 * 60 * 1000
    },


    // delay for visiting a link
    indexPageDelay: 5000,
    linkDelay: 2000,
    clickDelay: 1000,

    // force to initialize opening chrome
    init: true
};
<file_sep>/**
 * Created by jeanzhao on 10/23/18.
 */
export * from "./macrodownload";
<file_sep>import { Subject } from "rxjs";

declare interface LaunchOptions {
    /**
     * Whether to open chrome in appMode.
     * @default false
     */
    appMode?: boolean;
    /**
     * Whether to ignore HTTPS errors during navigation.
     * @default false
     */
    ignoreHTTPSErrors?: boolean;
    /**
     * Do not use `puppeteer.defaultArgs()` for launching Chromium.
     * @default false
     */
    ignoreDefaultArgs?: boolean | string[];
    /**
     * Whether to run Chromium in headless mode.
     * @default true
     */
    headless?: boolean;
    /**
     * Path to a Chromium executable to run instead of bundled Chromium. If
     * executablePath is a relative path, then it is resolved relative to current
     * working directory.
     */
    executablePath?: string;
    /**
     * Slows down Puppeteer operations by the specified amount of milliseconds.
     * Useful so that you can see what is going on.
     */
    slowMo?: number;
    /**
     * Sets a consistent viewport for each page. Defaults to an 800x600 viewport. null disables the default viewport.
     */
    defaultViewport?: {
        /**
         * page width in pixels.
         */
        width?: number;
        /**
         * page height in pixels.
         */
        height?: number;
        /**
         * Specify device scale factor (can be thought of as dpr).
         * @default 1
         */
        deviceScaleFactor?: number;
        /**
         * Whether the meta viewport tag is taken into account.
         * @default false
         */
        isMobile?: boolean;
        /**
         *
         * Specifies if viewport supports touch events.
         * @default false
         */
        hasTouch?: boolean;
        /**
         * Specifies if viewport is in landscape mode.
         * @default false
         */
        isLandscape?: boolean;
    };
    /**
     * Additional arguments to pass to the Chromium instance. List of Chromium
     * flags can be found here.
     */
    args?: string[];
    /**
     * Close chrome process on Ctrl-C.
     * @default true
     */
    handleSIGINT?: boolean;
    /**
     * Close chrome process on SIGTERM.
     * @default true
     */
    handleSIGTERM?: boolean;
    /**
     * Close chrome process on SIGHUP.
     * @default true
     */
    handleSIGHUP?: boolean;
    /**
     * Maximum time in milliseconds to wait for the Chrome instance to start.
     * Pass 0 to disable timeout.
     * @default 30000 (30 seconds).
     */
    timeout?: number;
    /**
     * Whether to pipe browser process stdout and stderr into process.stdout and
     * process.stderr.
     * @default false
     */
    dumpio?: boolean;
    /** Path to a User Data Directory. */
    userDataDir?: string;
    /**
     * Specify environment variables that will be visible to Chromium.
     * @default `process.env`.
     */
    env?: {
        [key: string]: string | boolean | number;
    };
    /**
     * Whether to auto-open DevTools panel for each tab. If this option is true, the headless option will be set false.
     */
    devtools?: boolean;
    /**
     * Connects to the browser over a pipe instead of a WebSocket.
     * @default false
     */
    pipe?: boolean;
}
declare interface linkInfo {
    /**
     * description for this download link:
     * could be set as widget name, action description
     */
    name: string;
    /**
     * selector for download body:
     * would click download link until the body is ready
     */
    waitSelector: string;
    /**
     * selector for possible error
     * could save much time instead of timeout for waitSelector
     */
    errorSelector: string;
    /**
     * selector for download link
     */
    downloadLink: string;
}
declare interface ERROR {
    type: string;
    info: string;
}


export interface Options {
    /**
     * site to visit
     */
    site: string;
    /**
     * launch options for using puppeteer
     * @default read from defaultConfig.ts file
     */
    launchOptions?: LaunchOptions;
    /**
     * whether to force to initialize opening chrome
     * @default read from defaultConfig.ts file
     * */
    init?: boolean;
    /**
     * delay in milliseconds to wait for the Chrome instance to start.
     * @default read from defaultConfig.ts file
     */
    indexPageDelay?: number;
    /**
     * delay in milliseconds to wait for the possible rendering time of link
     * @default read from defaultConfig.ts file
     */
    linkDelay?: number;
    /**
     * delay in milliseconds to wait for the possible responding time of link
     * @default read from defaultConfig.ts file
     */
    clickDelay?: number;
}
export interface Task {
    /**
     * task name:
     * special one for same macroId
     */
    name: string;
    /**
     * basis for calculating progress
     */
    macroId: string;
    /**
     *  menu selector list for sequentially click
     */
    menuList?: string[];
    /**
     * directory path for multiple download files
     */
    downloadPath: string;
    /**
     * a list of download link information for this menu link
     */
    downloadLink: linkInfo[];
    /**
     * option value set in widgets after clicking current menu link
     */
    param?: any;
}
export interface TaskStatus {
    /**
     * task name:
     * special one for same macroId
     */
    name: string;
    /**
     * process info for the browser
     */
    processInfo: ProcessInfo;
    /**
     * basis for calculating progress
     */
    macroId: string;
    /**
     *  total number for downloading links
     */
    total: number;
    /**
     * successful number for downloading links
     */
    success: number;
    /**
     * a list of error info during downloading files
     */
    errInfo: ERROR[];
    /**
     * other reported error case:
     * e.g. the browser is closed manually.
     */
    otherErr: ERROR[];
}
export interface ProcessInfo {
    pid: number;
    cpu: number;
}

export declare class MacroDownload {
    runningSteam: Subject<TaskStatus[]>;
    constructor();
    static getCustomizedOptions(options: Options): Options;
    download(tasks: Task[], options: Options): Promise<void>;
    close(): Promise<void>;
    getCpuProcess(processId?: number): Promise<ProcessInfo>;
    getRunningProcess(): any;
}
<file_sep>/**
 * Created by jeanzhao on 10/23/18.
 */
export * from "./job/MacroDownload";<file_sep>/**
 * Created by jeanzhao on 11/23/18.
 */
import {LaunchOptions} from "../model/Options";
import * as winston from "winston";

export class UtilService {
    private static _instance: UtilService = new UtilService();
    constructor(){
        UtilService._instance = this;
    }
    static isNull(i: any): boolean {
        return Object.prototype.toString.call(i) === "[object Null]";
    }

    static isUndefined(i: any): boolean {
        return Object.prototype.toString.call(i) === "[object Undefined]";
    }

    static isString(i: any): boolean {
        return Object.prototype.toString.call(i) === "[object String]";
    }

    static isEqualValue(p1: LaunchOptions, p2: LaunchOptions): boolean {
        let result;
        for (let k in p1) {
            result = p1[k] !== p2[k];
        }
        return result;
    }

    delay(time: number): Promise<string> {
        return new Promise(resolve => {
            setTimeout(() => resolve(""), time);
        });
    }

    mapSeries(array: any[], promise: (value, index) => Promise<any>): Promise<any> {
        if (array.length > 0) {
            return array.reduce((p, c, i) => p.then(() => promise(c, i)), Promise.resolve());
        } else {
            winston.info(`found empty list need execution.`);
            return Promise.resolve();
        }
    }

}<file_sep>/**
 * Created by jeanzhao on 11/23/18.
 */

export interface TaskStatus {
    /**
     * task name:
     * special one for same macroId
     */
    name: string;
    /**
     * process info for the browser
     */
    processInfo: ProcessInfo;

    /**
     * basis for calculating progress
     */
    macroId: string;
    /**
     *  total number for downloading links
     */
    total: number;
    /**
     * successful number for downloading links
     */
    success: number;
    /**
     * a list of error info during downloading files
     */
    errInfo: ERROR[];
    /**
     * other reported error case:
     * e.g. the browser is closed manually.
     */
    otherErr: ERROR[];
}

export interface ERROR {
    type: string,
    info: string
}

export interface ProcessInfo {
    pid: number,
    cpu: number
}<file_sep># macro-execution
This is a simple plug-in based on puppeteer.   
  
Currently it supports macro download. It helps monitoring tasks status for macro downloading files from a virtual web page.    
It also provides customized settings and pushes download progress for per task.  

Please feel free to use it in javascript or typescript projects.  

## Install
 * npm install macro-execution

## Usage
### Used in js projects:   
```js
let me = require(`macro-execution`); 

function A(tasks){
  let test = new me.MacroDownload();
  return test.download(tasks, {site: 'localhost:3000'})
      .then(test.close);
}

let tasks = [{
  name: '<NAME>',       // task name
  menuList: [],            // nested menu wait to be clicked
  downloadPath: './Test',  // the directory to download files
  downloadLink: [],        // download link set, e.g.
                           // linkInfo {
                           //       name: ''
                           //       waitSelector: ''    // selector for download body
                           //       errorSelector: ''   // selector for possible error
                           //       downloadLink: ''    // selector for download link
                           //   }
  macroId: 'q1'            // unique id for running tasks
}];

A(tasks);
``` 
### Used in ts projects: 
```ts
import {Task, MacroDownload} from 'macro-execution';

export function A(tasks: Task[]): Promise<void> {
    let test = new MacroDownload();
    return test.download(tasks, {site: 'http://localhost:3000'})
        .then(test.close);
}
``` 
## Launch options

| Parameters | default Value | Descriptions   
| ---        | ---          | ---       |
| `site`     | `--`         | `must be set: site to visit` 

## API

* Disconnect  
```js  
test.close();  
```

* Monitoring task status  

```js 
test.runningSteam.subscribe(list => {  

// do something with ${list}  

// [{  "name":"<NAME>",  
//     "processInfo": {"pid":25459,"cpu":0},   // process id and cpu info  
//     "macroId":"q1",                         // unique id for running tasks  
//     "total":0,                              // total number for per task   
//     "success":0,                            // current success number for per task   
//     "errInfo":[],                           // error message  
//     "otherErr":[]                           // unexpected error message collection   
// }]   
});  
```

# Coming soon  

More supports is coming...    

It's for open source learning.   
Any questions please feel free to contact me.
<file_sep>/**
 * Created by jeanzhao on 10/23/18.
 */


export interface Task {
    /**
     * task name:
     * special one for same macroId
     */
    name: string;
    /**
     * basis for calculating progress
     */
    macroId: string;
    /**
     *  menu selector list for sequentially click
     */
    menuList?: string[];
    /**
     * directory path for multiple download files
     */
    downloadPath: string;
    /**
     * a list of download link information for this menu link
     */
    downloadLink: linkInfo[];
    /**
     * option value set in widgets after clicking current menu link
     */
    param?: any;
}

export interface linkInfo {
    /**
     * description for this download link:
     * could be set as widget name, action description
     */
    name: string;
    /**
     * selector for download body:
     * would click download link until the body is ready
     */
    waitSelector: string;
    /**
     * selector for possible error
     * could save much time instead of timeout for waitSelector
     */
    errorSelector: string;
    /**
     * selector for download link
     */
    downloadLink: string;

}<file_sep>/**
 * Created by jeanzhao on 12/23/18.
 */
import * as puppeteer from "puppeteer";
import {UtilService} from "../Services/utils";
import {Subject} from "rxjs";
import {Options, LaunchOptions} from "../model/Options";
import {DefaultConfig} from "../model/defaultConfig";
import {Task} from "../model/Task";
import {ERROR, ProcessInfo, TaskStatus} from "../model/TaskStatus";
import * as winston from "winston";
import * as pidUsage from 'pidusage';

export class MacroDownload {
    private utils = new UtilService();
    private browser: any;
    private launch: LaunchOptions;
    private runningList: TaskStatus[] = [];

    public runningSteam: Subject<TaskStatus[]> = new Subject<TaskStatus[]>();

    constructor() {
    }

    static getCustomizedOptions(options: Options): Options {
        let result: any = Object.assign({}, DefaultConfig);
        if (options.launchOptions) {
            Object.assign(result.launchOptions, options.launchOptions);
        }
        let remainingConf = Object.assign({}, options);
        for (let attr in remainingConf) {
            if (attr === 'launchOptions') delete remainingConf[attr];
        }
        Object.assign(result, remainingConf);
        return result;
    }

    public async close() {
        await this.utils.delay(DefaultConfig.indexPageDelay);
        await this.browser.close();
        await pidUsage.clear();
        this.browser = null;
        this.launch = null;
        this.runningSteam = null;
        return;
    }

    public async getCpuProcess(processId?: number): Promise<ProcessInfo> {
        let pid = processId || this.getRunningProcess().pid;
        let cpu = 0;
        if (pid) {
            let stat = await pidUsage(pid);
            cpu = stat.cpu ? +(stat.cpu).toFixed(2) : 0;
        }
        return {pid, cpu};
    }

    public getRunningProcess() {
        return this.browser ? this.browser.process() : {};
    }

    public async initTaskList(tasks: Task[]) {
        const processInfo = await this.getCpuProcess();
        tasks.forEach(task => {
            const index = this.runningList.findIndex(t => t.name === task.name && t.macroId === task.macroId);
            if (index !== -1) {
                this.runningList[index].total = +this.runningList[index].total + task.downloadLink.filter(i => !!i.downloadLink).length;
            } else {
                this.runningList.push({
                    "name": task.name,
                    "processInfo": processInfo,
                    "macroId": task.macroId,
                    "total": task.downloadLink.filter(i => !!i.downloadLink).length,
                    "success": 0,
                    "errInfo": [],
                    "otherErr": []
                });
            }
        });
        this.runningSteam.next(this.runningList);
        return;
    }

    public async download(tasks: Task[], options: Options) {
        let page = await this.openWebPage(options,
            () => this.initTaskList(tasks)
        );
        winston.info(`start macro download, task numbers:${tasks.length}`);
        await this.utils.mapSeries(tasks, async task => {
            try {
                await (page as any)._client.send('Page.setDownloadBehavior', {
                    behavior: 'allow',
                    downloadPath: task.downloadPath
                });

                let successIndex = 0;
                const total = task.downloadLink.filter(i => !!i.downloadLink).length;

                // 1. click nested menu
                await this.utils.mapSeries(task.menuList, async (menuSelector, index) => {
                    winston.info(`click menu with selector:${menuSelector}`);
                    await this.clickBySelector(menuSelector, task, page);
                });

                // 2. download files after the last menu link
                await this.utils.mapSeries(task.downloadLink, async linkInfo => {
                    winston.info(`start to download a file for ${linkInfo.name}`);
                    try {
                        await this.raceCheck(linkInfo.waitSelector, linkInfo.errorSelector, page);
                        let downloadContent = await this.raceCheck(linkInfo.downloadLink, linkInfo.errorSelector, page);
                        await this.clickHandler(downloadContent);
                        successIndex++;
                        this.pushList({task: task, success: 1});
                        winston.debug(`Complete ${successIndex}/${total} for downloading ${task.name}${task.param ? ' with param ' + JSON.stringify(task.param) : ''}`);
                        await this.utils.delay(DefaultConfig.clickDelay);
                    } catch (e) {
                        const info = `fail download file for ${linkInfo.name} of link '${task.name}'${task.param ? ' with param ' + JSON.stringify(task.param) : ''}`;
                        this.errHandler(task, e.message ? e : info);
                    }
                });
            } catch (e) {
                this.errHandler(task, e);
            }
        });
        return;
    }

    private async openWebPage(options: Options, launchCallback?: () => void) {
        let conf = MacroDownload.getCustomizedOptions(options);
        if (this.isNeedLaunch(conf)) {
            this.launch = conf.launchOptions;
            this.browser = await puppeteer.launch(this.launch);
            if (launchCallback) {
                launchCallback();
            }
        }
        let page = await this.browser.newPage();
        await page.goto(`${conf.site}`);
        await this.utils.delay(conf.indexPageDelay);
        return page;
    }

    private isNeedLaunch(options: Options): boolean {
        return UtilService.isNull(this.browser) || UtilService.isUndefined(this.browser)
            || (options.init && this.launch
                && !UtilService.isEqualValue(this.launch, options.launchOptions));
    }

    private async pushList(item: { task: Task, success?: number, err?: ERROR }) {
        const index = this.runningList.findIndex(t => t.name === item.task.name && t.macroId === item.task.macroId);
        if (index !== -1) {
            let curLink = this.runningList[index];
            if (item.success) {
                curLink["success"] = +curLink["success"] + item.success;
            }

            if (item.err) {
                const errKey = item.err.type === "download" ? "err" : "otherErr";
                item.err.info = `[${curLink.macroId}]: ${item.err.info}`;
                if (!errKey.includes(item.err.info)) {
                    curLink[errKey].push(item.err.info);
                }
            }

            curLink.processInfo = await this.getCpuProcess(curLink.processInfo.pid);
            this.runningSteam.next(this.runningList);
        }
    }

    private async clickBySelector(selector: string, task: Task, page: any) {
        try {
            let curMenu = await page.waitForSelector(selector, {timeout: this.launch.timeout});
            await this.clickHandler(curMenu);
            return;
        } catch (e) {
            this.errHandler(task, e);
        }
    }

    private raceCheck(selector: string, errorSelector: string, page: any): Promise<any> {
        return Promise.race([this.checkSelector(selector, page), this.checkSelector(errorSelector, page)]);
    }

    private checkSelector(selector: string, page: any): Promise<any> {
        return page.waitForSelector(selector, {timeout: this.launch.timeout})
            .then(value => {
                return selector.includes('img[src$=\'/error_500.png\']') ? Promise.reject() : Promise.resolve(value);
            });
    }

    private async clickHandler($selector: any) {
        await $selector.click({clickCount: 1, delay: DefaultConfig.clickDelay});
        await this.utils.delay(DefaultConfig.clickDelay);
        return;
    }

    private errHandler(task: Task, e: any) {
        let err;
        if (UtilService.isString(e)) {
            err = {"type": "download", "info": e};
        } else {
            err = {"type": "other", "info": e.message || JSON.stringify(e)};
        }
        this.pushList({task: task, err: err});
        return;
    }

}
