<file_sep>-- file: src/sql/pgsql/qto/app-itms/003.create-table.app_items_roles_permissions.sql
-- v0.8.4

-- \echo 'If necessary, perform -- DROP TABLE IF EXISTS app_items_roles_permissions;' 

-- \echo '4. Creating the app_items_roles_permissions table'

CREATE TABLE app_items_roles_permissions (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             BIGSERIAL UNIQUE NOT NULL 
    , app_roles_guid     UUID NOT NULL DEFAULT gen_random_uuid()
    , allowed        bool default true
    , app_routes_guid UUID NOT NULL DEFAULT gen_random_uuid()
    , app_items_guid UUID NOT NULL DEFAULT gen_random_uuid()
    , name           varchar (100) NOT NULL DEFAULT 'name...'
    , description    varchar (200) NULL DEFAULT 'description...'
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_app_items_roles_permissions_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

create unique index idx_app_items_roles_permissions_uniq_id on app_items_roles_permissions (id);


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.app_items_roles_permissions'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 

CREATE TRIGGER trg_set_update_time_on_app_items_roles_permissions BEFORE UPDATE ON app_items_roles_permissions 
   FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'app_items_roles_permissions'::regclass;

<file_sep>-- file: src/sql/pgsql/qto/tables/40.definitions_dictionary.sql
-- v0.7.8
-- DROP TABLE IF EXISTS definitions_dictionary CASCADE;

SELECT 'create the "definitions_dictionary" table'
; 
   CREATE TABLE definitions_dictionary (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , name           varchar (100) NOT NULL DEFAULT 'name...'
    , description    varchar (4000) DEFAULT 'description...'
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , listing_id     varchar (20) NULL DEFAULT 'runtime...'
    , CONSTRAINT pk_definitions_dictionary_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

   create unique index idx_uniq_definitions_dictionary_id on definitions_dictionary (id);



SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.definitions_dictionary'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 

--The trigger:
   CREATE TRIGGER trg_set_update_time_on_definitions_dictionary 
   BEFORE UPDATE ON definitions_dictionary 
   FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'definitions_dictionary'::regclass;

<file_sep>SELECT 'create the "bugs" table'
; 

CREATE TABLE public.bugs (
    guid uuid DEFAULT public.gen_random_uuid() NOT NULL,
    id bigint DEFAULT (to_char(CURRENT_TIMESTAMP, 'YYMMDDHH12MISS'::text))::bigint NOT NULL,
    prio integer DEFAULT 0 NOT NULL,
    issues_status_guid uuid DEFAULT 'cb989a14-d0b8-46e4-b2cc-5e2a974b5d29'::uuid NOT NULL,
    name character varying(100) DEFAULT 'name ...'::character varying NOT NULL,
    description character varying(4000) DEFAULT 'desc ...'::character varying NOT NULL,
    solution_proposal character varying(4000),
    location character varying(100),
    app_users_guid uuid DEFAULT '2660a6e9-9e6b-4faa-8264-27a92872657b'::uuid NOT NULL,
    category_guid uuid DEFAULT '70724562-d83c-446e-94cf-58ced84f3a0e'::uuid NOT NULL,
    update_time timestamp without time zone DEFAULT date_trunc('second'::text, now()) NOT NULL
    , CONSTRAINT pk_bugs_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.bugs'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 


--The trigger:
CREATE TRIGGER trg_set_update_time_on_bugs BEFORE UPDATE ON bugs FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'bugs'::regclass;
<file_sep>place here any binaries (both produced by the apps in your qto and external ones) you want to distribute with your qto.
<file_sep>doMojoHypnotoadStart(){
   
   do_export_json_section_vars $PRODUCT_DIR/cnf/env/$ENV.env.json '.env.app'
   doMojoHypnotoadStop

   export MOJO_LOG_LEVEL='error'; #could be warn, debug, info
   export MOJO_MODE='production'; #could be development as well ...

   while read -r p ; do 
      p=$(echo $p|sed 's/^ *//g')
      test $p -ne $$ && kill -9 $p ; 
   done < <(sudo ps -ef | grep -v grep | grep 'mojo-hypnotoad-start'|awk '{print $2}')

   cd $PRODUCT_DIR/src/perl/qto/script

   hypnotoad qto &
   cd $PRODUCT_DIR
   sudo service nginx restart
   test $? -ne 0 && exit 1
   sudo ps -ef | grep -v grep | grep qto
}

<file_sep># ssh public private key authentication 

ssh-keygen -t rsa -b 4096 -C "<EMAIL>"

cat id_rsa.pub >> ~/.ssh/authorized_keys
cat ~/.ssh/authorized_keys
chmod -v 0700 ~/.ssh
chmod -v 0600 ~/.ssh/authorized_keys
chmod -v 0600 ~/.ssh/id_rsa
chmod -v 0644 ~/.ssh/id_rsa.pub
find ~/.ssh -exec stat -c "%U:%G %a %n" {} \;
rm -fv ~/id_rsa.pub
exit
# and verify that you can go on the server without having to type a pass
ssh $ssh_user@$ssh_server
# STOP COPY
# START === how-to implement public private key ( pkk ) authentication 
# create pub priv keys on server
# START copy 
ssh-keygen -t rsa -b 4096 -C "<EMAIL>"
ssh-keygen -t rsa
ssh-keygen -t rsa -b 4096 -C "<EMAIL>" -f ~/.ssh/id_rsa.pub
ssh-keygen -y -f ~/.ssh/id_rsa > ~/.ssh/id_rsa.pub
# Hit enter twice 
# copy the rsa pub key to the ssh server
scp ~/.ssh/id_rsa.pub $ssh_user@$ssh_server:/home/$ssh_user/

<file_sep># v0.2.1
#------------------------------------------------------------------------------
#  gmail the latest created package - requires mutt binary !!!
#------------------------------------------------------------------------------
do_gmail_package(){

	mutt --help >/dev/null 2>&1 ||
	{ do_log "ERROR. mutt is not installed or not in PATH. Aborting." >&2; exit 1; }

   if [ -z "$AdminEmail" ]; then
      msg="AdminEmail to set mail to not set !!! you need to export AdminEmail=list-of-emails-to-send-mail-to-comma-delimited" ;
      export exit_code=1 ;
      do_exit "$msg";
      exit 1 ;
   fi

	# zip_file=$(ls -r1 "$product_dir"/*.zip | head -1)
	zip_file=$(cat $PRODUCT_DIR/dat/$RUN_UNIT/tmp/zip_file)
   test -z "${zip_file+x}" && zip_file=$(ls -r1 "$PRODUCT_DIR"/*.zip | head -1)
   test -z "${zip_file+x}" && export exit_code=1 && do_exit " no zip file found for gmailing !!!"

   zip_file_name=`basename $zip_file`

	# create a fake txt file type attachment
	cp -v "$zip_file" "$zip_file"'.txt'

	echo $zip_file>$tmp_dir/.msg

   do_log "DEBUG cnf_files is ${cnf_file+x}"
   do_log "DEBUG AdminEmail: $AdminEmail"

	for Email in $AdminEmail; do (
		mutt -s "${mail_msg-}::: ""$zip_file_name" -a "$zip_file"'.txt' -- "$Email" < $tmp_dir/.msg
	);
	done

	rm -fv "$zip_file"'.txt'

}
<file_sep># usage: include it in your Makefile
# include lib/make/make-help.task

.DEFAULT_GOAL := help

.PHONY: help  ## @-> show this help  the default action
help:
	@clear
	@fgrep -h "##" $(MAKEFILE_LIST)|fgrep -v fgrep|sed -e 's/^\.PHONY: //'|sed -e 's/^\(.*\)##/\1/'| \
      column -t -s $$'@'
<file_sep>if / when the apps of your qto generate data place into subdirs of this dir ...
<file_sep>SELECT column_name
FROM information_schema.columns
WHERE 1=1
AND table_schema = 'public'
AND table_name   =:'table_name'  ;
--SELECT datname , datcollate FROM pg_database WHERE datname=:'postgres_app_db' ; 
/*
# or straight cp to bash

export postgres_app_db='my_db'
export table_name='my_table_name'

while read -r c; do test -z "$c" || echo $table_name.$c , ; done < <(cat << EOF | psql -t -q -d $postgres_app_db -v table_name="${table_name:-}"
SELECT column_name
FROM information_schema.columns
WHERE 1=1
AND table_schema = 'public'
AND table_name   =:'table_name'  ;
EOF
)

while read -r c; do test -z "$c" || echo  , $table_name.$c  | perl -ne 's/\n//gm;print' ; done < <(cat << EOF | psql -t -q -d $postgres_app_db -v table_name="${table_name:-}"
SELECT column_name
FROM information_schema.columns
WHERE 1=1
AND table_schema = 'public'
AND table_name   =:'table_name'  ;
EOF
)


while read -r c; do test -z "$c" || echo  $c = excluded.$c , | perl -ne 's/\n//gm;print' ; done < <(cat << EOF | psql -t -q -d $postgres_app_db -v table_name="${table_name:-}"
SELECT column_name
FROM information_schema.columns
WHERE 1=1
AND table_schema = 'public'
AND table_name   =:'table_name'  ;
EOF
)
*/
<file_sep> ---- DROP TABLE IF EXISTS tst_paging ; 

SELECT 'create the "tst_paging" table'
; 
   CREATE TABLE tst_paging (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , prio           integer NOT NULL
    , name           varchar (200) NOT NULL
    , CONSTRAINT pk_tst_paging_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

create unique index idx_tst_paging_uniq_id on tst_paging (id);


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.tst_paging'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 
<file_sep># src/bash/qto/funcs/remove-action-files.test.sh

# v1.1.2
# ---------------------------------------------------------
# adds first an action to remove 
# generates all the aciton files (( it will add this new ) 
# action to remove
# and tests the actual removal at the end 
# ---------------------------------------------------------
doTestRemoveActionFiles(){

	source $PRODUCT_DIR/lib/bash/funcs/flush-screen.sh
	do_log "DEBUG START doTestRemoveActionFiles"

	doSpecRemoveActionFiles
	sleep "$sleep_interval"
	do_flush_screen

	doHelpRemoveActionFiles
	sleep "$sleep_interval"
	do_flush_screen
	
	cat doc/txt/qto/tests/remove-action-files.test.txt
	sleep "$sleep_interval"
	do_flush_screen
	
	# add an action to remove
	found=$(grep -c action-to-remove src/bash/qto/tests/rem-qto-actions.lst)
	test $found -eq 0 && \
	echo action-to-remove >> src/bash/qto/tests/rem-qto-actions.lst
	found=0
	
	found=$(grep -c action-to-remove src/bash/qto/tests/all-qto-tests.lst)
	test $found -eq 0 && \
		echo action-to-remove >> src/bash/qto/tests/all-qto-tests.lst

	# now generate the code files for this action to remove
	bash src/bash/qto/qto.sh -a generate-action-files		
	sleep "$sleep_interval"
	do_flush_screen

	# and test the actual removal of the action 	
	bash src/bash/qto/qto.sh -a remove-action-files		
	do_log "DEBUG STOP  doTestRemoveActionFiles"

	sleep "$sleep_interval"

	do_flush_screen
}
# eof func doTestRemoveActionFiles


# eof file: src/bash/qto/funcs/remove-action-files.test.sh
<file_sep>.PHONY: check_git  ## @-> checks if your branch is ahead of origin/master branc
check_git:
	@echo "Checking if your branch is ahead of origin/master branch..."
	@if [ "$(shell git rev-list --left-right --count origin/master..HEAD | cut -f1|xargs)" != "0" ]; then \
		echo "You current branch is behind origin/master branch. Please merge/rebase your branch and try again!"; \
		exit 1; \
	fi
<file_sep>#  QTO INSTALLATIONS AND CONFIGURATION GUIDE
* [1. INTRODUCTION](#1-introduction)
  * [1.1. PURPOSE](#11-purpose)
  * [1.2. DOCUMENT STATUS](#12-document-status)
  * [1.3. AUDIENCE](#13-audience)
  * [1.4. MASTER STORAGE AND STORAGE FORMAT](#14-master-storage-and-storage-format)
  * [1.5. VERSION CONTROL](#15-version-control)
  * [1.6. PROCESS](#16-process)
* [2. LOCAL DEPLOYMENT](#2-local-deployment)
  * [2.1. PREREQUISITES](#21-prerequisites)
  * [2.2. TARGET SETUP](#22-target-setup)
  * [2.3. ENSURE YOU HAVE PASSWORDLESS SUDO ](#23-ensure-you-have-passwordless-sudo-)
  * [2.4. CLONE QTO REPOSITORY TO YOUR GITHUB ACCOUNT](#24-clone-qto-repository-to-your-github-account)
  * [2.5. INSTALL AND TMUX, VIM, GIT AND BASH CONFS (OPTIONAL)](#25-install-and-tmux-vim-git-and-bash-confs-(optional))
  * [2.6. INSTALL VB+VAGRANT VIRTUAL MACHINE FOR A LOCAL UBUNTU HOST (OPTIONAL) ](#26-install-vb+vagrant-virtual-machine-for-a-local-ubuntu-host-(optional)-)
  * [2.7. ADD TRUST BETWEEN LOCAL HOST AND GITHUB SERVER](#27-add-trust-between-local-host-and-github-server)
  * [2.8. BOOTSTRAP AND DEPLOY THE APPLICATION LOCALLY](#28-bootstrap-and-deploy-the-application-locally)
  * [2.9. PROVISION THE APPLICATION LOCALLY](#29-provision-the-application-locally)
  * [2.10. START THE APPLICATION LAYER ON THE DEV INSTANCE](#210-start-the-application-layer-on-the-dev-instance)
  * [2.11. CREATE AND START THE TST INSTANCE](#211-create-and-start-the-tst-instance)
  * [2.12. CHECK APPLICATION LAYER SYNTAX, LOAD DB DATA](#212-check-application-layer-syntax-load-db-data)
  * [2.13. START THE TST INSTANCE](#213-start-the-tst-instance)
* [3. FIRST TIME AWS DEPLOYMENT](#3-first-time-aws-deployment)
  * [3.1. AWS CENTRIC INFRASTRUCTURE AS A CODE RELATED INTRO](#31-aws-centric-infrastructure-as-a-code-related-intro)
  * [3.2. PREREQUISITES](#32-prerequisites)
    * [3.2.1. Configure the AdminEmail](#321-configure-the-adminemail)
    * [3.2.2. Create a pem file for deployment](#322-create-a-pem-file-for-deployment)
    * [3.2.3. Create AWS instance deployment keys](#323-create-aws-instance-deployment-keys)
    * [3.2.4. Configure your AWS credentials - AWS keys](#324-configure-your-aws-credentials--aws-keys)
  * [3.3. INITIALISE AWS INFRASTRUCTURE](#33-initialise-aws-infrastructure)
  * [3.4. SET IP ADDRESS FOR THE HOST IN DNS (OPTIONAL) ](#34-set-ip-address-for-the-host-in-dns-(optional)-)
  * [3.5. ACCESS AWS HOST VIA SSH](#35-access-aws-host-via-ssh)
  * [3.6. FETCH THE SOURCE CODE FROM GITHUB ON THE AWS SERVER](#36-fetch-the-source-code-from-github-on-the-aws-server)
  * [3.7. BOOTSTRAP AND DEPLOY THE APPLICATION ON AWS INSTANCE](#37-bootstrap-and-deploy-the-application-on-aws-instance)
  * [3.8. PROVISION APPLICATION IN AWS INSTANCE IN DEV](#38-provision-application-in-aws-instance-in-dev)
  * [3.9. EDIT THE CONFIGURATION FILE AND START WEB SERVER](#39-edit-the-configuration-file-and-start-web-server)
  * [3.10. ACCESS THE QTO APPLICATION FROM WEB](#310-access-the-qto-application-from-web)
  * [3.11. CONFIGURE DNS](#311-configure-dns)
  * [3.12. PROVISION QTO WEB USERS](#312-provision-qto-web-users)
  * [3.13. CREATE THE TST PRODUCT INSTANCE](#313-create-the-tst-product-instance)
  * [3.14. PROVISION THE TST DATABASE](#314-provision-the-tst-database)
  * [3.15. FORK PRODUCTION INSTANCE](#315-fork-production-instance)
  * [3.16. PROVISION PRD DB](#316-provision-prd-db)
* [4. PROVISION HTTPS (ONLY IF DNS IS CONFIGURED)](#4-provision-https-(only-if-dns-is-configured))
  * [4.1. FORK LATEST STABLE DEV TO TST](#41-fork-latest-stable-dev-to-tst)
  * [4.2. GO TO YOUR PREVIOUS ENVIRONMENT](#42-go-to-your-previous-environment)
* [5. NON-FIRST TIME AWS DEPLOYMENT](#5-non-first-time-aws-deployment)
  * [5.1. CREATE AWS INSTANCE](#51-create-aws-instance)
  * [5.2. SETUP BASH & VIM](#52-setup-bash-&-vim)
  * [5.3. CLONE PROJECT TO GUEST SERVER](#53-clone-project-to-guest-server)
* [6. PHYSICAL HOST OS INSTALLATIONS](#6-physical-host-os-installations)
  * [6.1. MACOS](#61-macos)
    * [6.1.1. Install QtPass](#611-install-qtpass)
* [7. POTENTIAL PROBLEMS AND TROUBLESHOOTING](#7-potential-problems-and-troubleshooting)
  * [7.1. POSTGRES ADMIN USER PASSWORD IS WRONG](#71-postgres-admin-user-password-is-wrong)
  * [7.2. AWS WAS NOT ABLE TO VALIDATE THE PROVIDED ACCESS CREDENTIALS](#72-aws-was-not-able-to-validate-the-provided-access-credentials)
  * [7.3. REDIS REFUSES TO START ](#73-redis-refuses-to-start-)
  * [7.4. COULD NOT CONNECT TO REDIS SERVER](#74-could-not-connect-to-redis-server)
  * [7.5. YOU HAVE REACHED THE HW PROVISIONING LIMITS OF YOUR AWS ACCOUNT](#75-you-have-reached-the-hw-provisioning-limits-of-your-aws-account)
  * [7.6. CANNOT LOGIN IN WEB INTERFACE WITH ADMIN USER](#76-cannot-login-in-web-interface-with-admin-user)
  * [7.7. MISMATCH IN THE AWS](#77-mismatch-in-the-aws)
  * [7.8. THE PROBLEM OCCURRED IS NOT MENTIONED HERE ?!](#78-the-problem-occurred-is-not-mentioned-here-)




    

## 1. INTRODUCTION
The QTO installation could be compressed to four oneliners, yet if you are installing it for the first time you should read this guide thoroughly and CAREFULLY or at least jump through the code sections from the top till the bottom, simply because the stack is huge - PostgreSQL, Perl application layer with a lot of modules and dependencies and you will be deploying the whole stack.

    

### 1.1. Purpose
The purpose of this document is to provide a description for the tasks and activities to be performed in order to achieve a fully operational QTO application instance, such as the one running on the https://qto.fi site. 

    

### 1.2. Document status
This document is reviewed for the current QTO version it is generated from. The document is updated and reviewed after each QTO release.

    

### 1.3. Audience
This document is aimed for everyone, who shall, will or even would like to install a QTO application instance. Although this guide is aimed to be fully implementable via copy paste by following top to bottom, you need to have basic understanding of networking, protocols and Linux in order to complete the full installation, as your mileage will vary.
This document might be of interest for people from architectural or even business app_roles to the extend to verify the claims for easy deployability states in other parts of the QTO application documentation set.

    

### 1.4. Master storage and storage format
The master storage of this document is the master branch of the latest QTO release. The main storage format is Markdown. 

    

### 1.5. Version control
The contents of this document ARE updated according to the EXISTING features, functionalities and capabilities of the QTO version, in which this document residues. Thus the git commit hash of a release QTO version contains the md format of this document.

    

### 1.6. Process
The QTO provides a mean for tracking of this documentation contents to the source code per feature/functionality, thus should you find inconsistencies in the behaviour of the application and the content of this document, please create a bug issue and assign it to the owner of your product instance.

    

## 2. LOCAL DEPLOYMENT


    

### 2.1. Prerequisites
You need hardware, which is powerful enough to run a virtual machine with at least 2GB of RAM, preferably more, and at least 20GB of hard disk space. 

    

### 2.2. Target setup
The target setup of both the LOCAL an the AWS deployments is a system comprised of dev, tst, qas and prd instances of QTO locally and dev, tst, qas and prd instances in AWS. 
The target setup in this section is the "satellite-host" depicted as "dev-mac" in the below diagram.
The following diagram illustrates that setup. Naturally you will be deploying only the dev.&lt;&lt;site&gt;&gt;.com, when performing the installation for the first time. Thus, in the spirit of QTO, you will be moving fast and destroying in dev, reinforcing skills in tst and do it at once in prd.


    

### 2.3. Ensure you have passwordless sudo 
The next command check whether or not your Linux user can execute sudo commands without having to provide password by checking whether or not the:
"&lt;&lt;your-linux-accont&gt;&gt; ALL=(ALL) NOPASSWD: ALL"
line without the quotes is found in your /etc/sudoers file and if it is not it adds it to it. Make sure you copy paste the whole line, because if you brake the syntax of your /etc/sudoers file you would have to boot in safe mode and manually add the line into your /etc/sudoers file. 

    test $(grep `whoami` /etc/sudoers|wc -l) -ne 1 && echo $(whoami)' ALL=(ALL) NOPASSWD: ALL'|sudo tee -a /etc/sudoers

### 2.4. Clone QTO repository to your GitHub account
Fork the QTO repository to your GitHub account from the GitHub UI by clicking on the fork button. After several seconds you should be able to access the qto project instead from an url like this: 
https://github.com/YordanGeorgiev/qto
from the 
https://github.com/%YourGitHubUserName%/qto

This action will enable the Merge Requests creation according to the qto development process described in the devops guide out of the box.

    

### 2.5. Install and tmux, vim, git and bash confs (optional)
This step is optional - it will deploy tmux, vim and git with some default configurations, which you could omit. Replace the email and the initials passed as $1 and $2 to the script with your email and initials and not those below.

    # on the AWS server
    export your_git_user_name=YordanGeorgiev
    export your_email=<EMAIL>
    export your_initials=ysg
    
    curl https://raw.githubusercontent.com/$your_git_user_name/qto/master/src/bash/qto/scripts/setup-bash-tmux-vim.sh | bash -s $your_email $your_initials
    
    # on the AWS server
    mkdir -p ~/opt; cd $_ ;  git clone <EMAIL>:<<YourGitHubUserName>>/qto.git; cd ~/opt

### 2.6. Install VB+Vagrant virtual machine for a local Ubuntu host (optional) 
If your physical host OS is not Ubuntu 18.04 OS, you could install the latest version of the Oracle VirtualBox with Guest additions and HashiCorp Vagrant and run the deploy-vagrant-vm.sh bootstrapper script on the Host to get a fully provisioned Guest setup with all the binaries and configurations. This step might take around 30 min depending on the hardware of your host machine and the VirtualBox configurations in your setup.

     # in the shell of your OS bash shell of your host machine
    mkdir -p ~/opt; cd $_ ; git clone <EMAIL>:<<YourGitHubUserName>>/qto.git ; cd ~/opt/qto
    
    # run the vagrant provisioning script
    ./src/bash/deploy-vagrant-vm.sh
    
    # now go to the vagrant guest
    vagrant ssh
    
    # go to the qto project in the 
    cd /home/vagrant/opt/qto/

### 2.7. Add trust between local host and github server
To add the trust between the locahost andl the GitHub server you must paste your public ssh keys into the GitHub UserSettings , as described in the following link:
https://www.inmotionhosting.com/support/website/ssh/how-to-add-ssh-keys-to-your-github-account/

And it would look something as follows:

    # check the ~/.ssh dir, of course you will have a different output 
    find ~/.ssh|sort
    /home/ubuntu/.ssh
    /home/ubuntu/.ssh/authorized_keys
    /home/ubuntu/.ssh/id_rsa.ysg.ip-10-0-46-110
    /home/ubuntu/.ssh/id_rsa.ysg.ip-10-0-46-110.pub
    
    # get the full string of the public key of your local OS account
    # of course this line will look different to you ... so don't copy it ... 
    cat /home/ubuntu/.ssh/id_rsa.ysg.ip-10-0-46-110.pub
    
    # copy carefully from EXACTLY the beginning of the next line till the end of the email address
    # of course YOUR public ssh key WILL look different
    ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC<KEY> <EMAIL>@<EMAIL>
    

### 2.8. Bootstrap and deploy the application locally
This step might not be needed if you used the deploy-vagrant-vm.sh vagrant provisioning script as described above, yet should the vagrant provisioning have failed you could run it on the Vagrant Guest once again and it WILL NOT if you have proper network connectivity and read/write access to your host file system from the Guest.
The bootstrap script will deploy ALL the required Ubuntu 18.04 binaries AND Perl modules as well as perform the needed configurations to enable the creation and load of the QTO database. This step will take 20 min at least. The bootstrapping script will:
  - install all the required binaries
  - install all the required Perl modules as non-root
  - install and provision PostgreSQL
  - install and provision the nginx proxy server 


Check the main method in the run.sh and uncomment entities you do not want to install locally, should you have any.

    # in the shell of your local Ubuntu box, skip this cmd if you are on vagrant
    mkdir -p ~/opt; cd $_ ;  git clone <EMAIL>:<<YourGitHubUserName>>/qto.git ; cd ~/opt
    
    # run the bootstrap script and IMPORTANT !!! reload the bash env
    ./qto/src/bash/deployer/run.sh ; bash ; 
    # copy this line too !!! to ensure you reload the bash from the ^^^ line

### 2.9. Provision the application locally
The run of the following "shell actions" will create the QTO database and load it with a snapshot of it's data from an SQL dump stored in s3. If you start getting a lot of Perl "cannot not find module" syntax check error, you probably did not reload the bash shell, by typing "bash" and hitting Enter in the previous step.

     # go to the product instance dir 
    source $(find . -name '.env') && cd qto/qto.$VERSION.$ENV.$USER
    
    # ensure application layer consistency, run db ddl's and load data from s3
    ./src/bash/qto/qto.sh -a check-perl-syntax -a scramble-confs -a provision-db-admin -a run-qto-db-ddl -a load-db-data-from-s3

### 2.10. Start the application layer on the dev instance
Start the application layer by issuing the following command. The hypnotoad starting script ALWAYS checks whether or not you have public and private files for the JWT auth and if you do not have them if will create them basically only once. So hit Enter or type a password, when executing the action for the first time.

    # ALWAYS run any shell-action from the product instance dir 
    cd ~/opt/qto/qto.$VERSION.$ENV.$USER
    
    # start the web server
    bash src/bash/qto/qto.sh -a mojo-hypnotoad-start

### 2.11. Create and start the tst instance
The development product instance dir is aimed for development on that machine. You would need at least a testing and production environments according to QTO's (and a lot more applications out there) best practices.

    bash ./src/bash/qto/qto.sh -a to-env=tst
    
    # now to to the TST product instance directory
    cd ~/opt/qto/qto.$VERSION.tst.$USER

### 2.12. Check application layer syntax, load DB data
You can concatenate multiple shell-actions with the -a &lt;&lt;shell-action-01&gt;&gt; -a &lt;&lt;shell-action-02&gt;&gt; and so on.
This actions will:
 - check the application layer syntax 
 - provision the DB admin credentials in Postgres 
 - run all the DDL's of the db
 - load the DB data from s3

    ./src/bash/qto/qto.sh -a check-perl-syntax -a provision-db-admin -a run-qto-db-ddl -a load-db-data-from-s3

### 2.13. Start the tst instance
The hypnotoad starting script ALWAYS checks whether or not you have public and private files for the JWT auth and, if you do not have them, it will create them only once. So hit Enter or type a password when executing the action for the first time.

    bash src/bash/qto/qto.sh -a mojo-hypnotoad-start

## 3. FIRST TIME AWS DEPLOYMENT
This section WILL provide you will with ALL required steps to get a fully functional instance of the QTO application in AWS in about 35 min with automated shell commands. Do just copy paste the commands into your shell. Do NOT cd into different directories - this deployment has been tested more than 40 times by multiple professionals and successfully by exactly reproducing those steps, yet the variables in such a complex deployment are many, thus should you encounter new issues do directly contact the owner of the instance you got this deployment package from, that is the person owning the GitHub repository you fetched the source code from.


    

### 3.1. AWS centric infrastructure as a code related intro

This installation is not truly idempotent, meaning that not all infra resources with the same names are deleted - you might need to go and manually delete objects from the AWS admin console, IF you are running this installation for more than two times on the same environments. If you are using the same AWS account for all your environments - that is dev, tst and prd - you can be sure that this installation has been tested for FULL logical and physical separation of your provisioned QTO resources by environment - simply said whatever you deploy to dev, will not brake anything in tst and/or prod.

    

### 3.2. Prerequisites
You should have a local instance of QTO as far as you could issue the terraform shell action - DB and site configuration are not must have for the AWS deployment to succeed.

    

#### 3.2.1. Configure the AdminEmail
The AdminEmail value stored in the cnf/env/*.env.json files is the e-mail of the QTO product instance owner - aka the only user being able to edit other users' details.


    

#### 3.2.2. Create a pem file for deployment
Point your browser at the following URL:
https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=&lt;&lt;your-region&gt;&gt;#KeyPairs:
Click on the create key-pair button. For the name of the file add the following : 
key-pair-qto-amzn. Download the file and save it into the ~/.ssh dir of the satellite host:

    ls -la ~/.ssh/key-pair-qto-amzn.pem
    chmod 0400 ~/.ssh/key-pair-qto-amzn.pem
    -r-------- 1 ysg vboxsf 1670 maali 30 11:00 /home/ysg/.ssh/key-pair-qto-amzn.pem

#### 3.2.3. Create AWS instance deployment keys
You will use those deployment keys later on when you later on ssh -i &lt;&lt;full-deployment-key&gt;&gt; ubuntu@&lt;&lt;instance-dns&gt;&gt;

    # run on the satellite host
    # create the dev instance deployment key
    # of course you should use your own e-mail and NOT mine ;o)
    ssh-keygen -t rsa -b 4096 -C "<EMAIL>" -f ~/.ssh/id_rsa.aws-ec2.qto.dev
    # hit enter twice preferably , to not set up passwords .. 
    
    # create the tst instance deployment key
    ssh-keygen -t rsa -b 4096 -C "<EMAIL>" -f ~/.ssh/id_rsa.aws-ec2.qto.tst
    # hit enter twice preferably , to not set up passwords .. 
    
    # create the prd instance deployment key
    ssh-keygen -t rsa -b 4096 -C "<EMAIL>" -f ~/.ssh/id_rsa.aws-ec2.qto.prd
    # hit enter twice preferably , to not set up passwords .. 

#### 3.2.4. Configure your AWS credentials - AWS keys
Generate NEW AWS access- and secret-keys https://console.aws.amazon.com/iam/home?region=&lt;&lt;YOUR-AWS-REGION&gt;&gt;#/security_credentials. 
Make sure both the aws_access_key_id and the aws_secret_access_key do NOT have the "+" and "-" chars in them, as the terraform binary will throw an error (chk: installations_doc-200330074044).
Store the keys in the QTO's development environment configuration file - the cnf/env/dev.env.json file. Either configure your AWS with the AWS configure command or add the YOUR AWS access key id and AWS secret key id to the ~/.aws/credentials, which should look something like this:


      cat ~/.aws/credentials
    [prd-qto]
    aws_access_key_id = <KEY>
    aws_secret_access_key = <KEY>
    
    [tst-qto]
    aws_access_key_id = <KEY>
    aws_secret_access_key = <KEY>
    
    [dev-qto]
    aws_access_key_id = <KEY>
    aws_secret_access_key = <KEY>

### 3.3. Initialise AWS infrastructure
To initialise the git AWS infrastructure you need to clone the QTO source code locally first. If you are repeating this task all over, you might need to remove from the AWS web UI duplicating VPCs and elastic IPs.

    # go to the product instance dir
    cd ~/opt/qto/qto.0.8.5.dev.$USER
    
    # apply the infra terraform in the src/terraform/tpl/qto/main.tf.tpl 
    clear ; bash qto/src/bash/qto/qto.sh -a init-aws-instance

### 3.4. Set IP address for the host in DNS (optional) 
If you have registered your own DNS name, you should configure now the public IP address found in the Amazon ec2 instances section of the newly created host to the dns name you have registered with your DNS provider, as it usually takes some time for the DNS to replicate (with the qto.fi domain it takes about 5 min max, some DNS providers suggest even hours to be reserved, your mileage might vary).
If you do not have a registered DNS, you could either use directly the IP address or the DNS name provided by amazon in the ec2 settings of the newly created host, in this case you should configure the same DNS in the cnf/etc/dev.env.json file (env-&gt;app-&gt;web_host variable) for nginx to be able to pick it in its own configuration.

    

### 3.5. Access AWS host via SSH
To access the AWS host via SSH you need to copy the provided elastic IP which was created by the terraform script. In your browser go to the following URL:
https://eu-west-1.console.aws.amazon.com/ec2/v2/home?region=&lt;&lt;YOUR-AWS-REGION&gt;&gt;#Instances:sort=instanceId
You should see a listing of your AWS instances one of which should be named dev-qto-ec2 (that is the development instance of the QTO application). Click on its checkbox. Search for the "IPv4 Public IP" string and copy the value of the IP.

    # run locally 
    ssh -i ~/.ssh/id_rsa.aws-ec2.qto.dev ubuntu@<<just-copied-IPv4-Public-IP>>
    # for example: 
    # ssh  -i ~/.ssh/id_rsa.aws-ec2.qto.dev ubuntu@192.168.3.11
    
    

### 3.6. Fetch the source code from GitHub on the aws server
If you have not yet forked the qto to your GitHub do it now by forking it from the product instance owner's GitHub account - to be able to submit changes to YOUR OWN git repository during the installation.
To fetch the source code from GitHub you need to have a full trust between the aws host and your GitHub account via public private ssh keys. In order to achieve that you need to put a public open-ssh key in the ssh keys section of your GitHub account in the GitHub web UI.
If you know how-to do this you could skip the following automated run, but if you don't know than first run this script and do all the steps in the following this instruction:
https://www.inmotionhosting.com/support/website/ssh/how-to-add-ssh-keys-to-your-github-account/



    # on the AWS server
    export your_git_user_name=YordanGeorgiev #the one visible 
    export your_email=<EMAIL>
    export your_initials=ysg
    
    # optionally run a provisioning script installing tmux, vim and automating the ssh keys creation
    curl https://raw.githubusercontent.com/$your_git_user_name/qto/master/src/bash/qto/scripts/setup-bash-tmux-vim.sh | bash -s $your_email $your_initials
    
    # if you know what tmux is start it now ... 
    tmux
    
    # on the AWS server
    my_private_ssh_key_file=~/.ssh/id_rsa.$your_initials.`hostname -s`
    
    # each time you issue the git command in THIS shell, it will be referring to your private key file
    # which is the trusted one by GitHub because you added it's public key file to your settings ...
    alias git='GIT_SSH_COMMAND="ssh -i $my_private_ssh_key_file" git'
    
    # now you must be able to clone the project , without being asked for passwords
    mkdir -p ~/opt; cd $_ ;  git clone <EMAIL>:$your_git_user_name/qto.git; cd ~/opt
    
    
    

### 3.7. Bootstrap and deploy the application on AWS instance
The bootstrap script will deploy ALL the required Ubuntu 18.04 binaries AND Perl modules as well as perform the needed configurations to enable the creation and load of the QTO database. This step will take at least 20 min. The bootstrap script will perform the following actions:
 - install all the required binaries
 - install all the required Perl modules as non-root
 - install and provision Postgres
 - install and provision the nginx proxy server

Copy paste the full command below - this is IMPORTANT !!!

    # run the bootstrap script and IMPORTANT !!! reload the bash shell 
    bash ./qto/src/bash/deployer/run.sh ; bash ; 
    # copy THIS line as well !!! Just to make sure the enter and ; from ^^^ line are copied too
    
    

### 3.8. Provision application in AWS instance in dev
The run of the following "shell actions" will create the QTO database and load it with a snapshot of its data from a SQL dump stored in s3. 

     # go to the product instance dir 
    source $(find . -name '.env') && cd qto/qto.$VERSION.$ENV.$USER
    
    # ensure application layer consistency, run db ddl's and load data from s3
    bash ./src/bash/qto/qto.sh -a check-perl-syntax -a scramble-confs -a provision-db-admin -a run-qto-db-ddl -a load-db-data-from-s3

### 3.9. Edit the configuration file and start web server
You need to set the web_host variable in the configuration files to the IP address or DNS name, if you have configured one for this IP address, otherwise your nginx configuration will be broken.
The following command will both start the Mojolicious hypnotoad server initiating the QTO application layer and the nginx reverse proxy, which will listen on the app-&gt;port port defined in the cnf/env/dev.env.json file.

    # start the web server
    ./src/bash/qto/qto.sh -a mojo-hypnotoad-start

### 3.10. Access the QTO application from web
The QTO web application is available at the following address
http://&lt;&lt;just-copied-IPv4-Public-IP&gt;&gt;:8078 This should redirect you to the dev_qto/login page - this is the end-point via Mojolicious over http (NOT safe).

http://&lt;&lt;just-copied-IPv4-Public-IP&gt;&gt;:78 This should redirect you to the dev_qto/login page - via the nginx proxy (SAFE)


    

### 3.11. Configure DNS
Configure the DNS server name in the UI of your DNS provider.

    

### 3.12. Provision QTO web users
Open the cnf/env/dev.env.json, change the env-&gt;AdminEmail with an e-mail you have access to. Restart the web servers as shown below. Login via web interface.

    # the start action performs restart as well, if the web servers are running
    ./src/bash/qto/qto.sh -a mojo-hypnotoad-start

### 3.13. Create the tst product instance
If you revisit the target architecture picture(@installations_doc-10), the actions so far have been only the installations of the dev instance, which you should have be now up and running. 
QTO is designed around the idea of developing in dev (aka doing things for the first time and possibly with some errors), testing in tst (more of a testing and configuration allowed, but not developing with minor errors) and prd (where no errors are allowed and everything should go smoothly). 
Thus by now you have achieved only the dev instance deployment.

    ./src/bash/qto/qto.sh -a to-env=tst

### 3.14. Provision the tst database
If you revisit the target architecture picture(installations_doc-10), the actions so far have been only the installations of the dev instance, which you should have be now up and running. 
QTO is designed around the idea of developing in dev (aka doing things for the first time and possibly with some errors), testing in tst (more of a testing and configuration allowed, but not developing with minor errors and prd (where no errors are allowed and everything should go smoothly). 
Thus by now you have achieved only the dev instance deployment

    

### 3.15. Fork production instance
The creation of this one should succeed at once, as it is performed exactly in the same way as the creation of the testing (tst) instance.
You "fork" the production instance by issuing the following command:

    # for the tst product instance into the prd product instance
    ./src/bash/qto/qto.sh -a to-env=prd
    
    # thus the 
    find ~/opt/csitea/qto/ -type d -mindepth 1 -maxdepth 
    
    /home/ubuntu/opt/csitea/qto/qto.0.7.8.dev.ysg
    /home/ubuntu/opt/csitea/qto/qto.0.7.8.tst.ysg
    /home/ubuntu/opt/csitea/qto/qto.0.7.8.prd.ysg

### 3.16. Provision prd DB
The creation of prd instance should succeed at once, as it is performed exactly in the same way as the creation of the testing (tst) instance.
You "fork" the production instance by issuing the following command:

    ./src/bash/qto/qto.sh -a check-perl-syntax -a scramble-confs -a provision-db-admin -a run-qto-db-ddl -a load-db-data-from-s3

## 4. PROVISION HTTPS (ONLY IF DNS is configured)
This section is separated, as it is optional. If you have configured DNS, you could provision https for the nginx server by issuing the following command:


    ./src/bash/qto/qto.sh -a provision-https
    
    sudo certbot --nginx -d <<your.organisation.com>>
    
    
    # you should get the following output 
    #IMPORTANT NOTES:
    # - Unable to install the certificate
    # - Congratulations! Your certificate and chain have been saved at:
    #  /etc/letsencrypt/live/qto.fi/fullchain.pem
    #   Your key file has been saved at:
    
    # now re-run the 
    bash src/bash/qto/qto.sh -a provision-nginx -a provision-https
    
    
    

### 4.1. Fork latest stable dev to tst
By this point all obvious bugs within the scope of THIS release MUST be cleared out, if that is NOT the case, you are wasting your time by gaining wrong velocity to demonstrate some new cool features, which you WILL pay for later.

    # on your local dev box go the latest stable environment
    /hos/opt/csitea/qto/qto.0.7.9.dev.ysg
    
    bash src/bash/qto/qto.sh -a to-env=tst
    
    # now prepare the configuration in this environment on your local dev box
    cp -r /hos/opt/csitea/qto/qto.0.7.9.tst.ysg /hos/opt/csitea/qto/qto.0.7.9.tst.deploy
    
    # ensure you copy your non-hash and SECRET configuration to this instance
    cp -v ~/.qto/cnf/* cnf/env/

### 4.2. Go to your previous environment
Go to your old environment: it contains your configuration at least you could spare yourself for copy paste for the creating of the RIGHT configuration for your ENTIRELY new deployment built with the infrastructure as a code.
Open the admin console:

    ssh -i /home/ysg/.ssh/id_rsa.prd.qto  ubuntu@ec2-52-209-247-245.eu-west-1.compute.amazonaws.com

## 5. NON-FIRST TIME AWS DEPLOYMENT
This section is aimed for the fortuned folks, who have already deployed at least one fully functional up-and-running instance of qto, thus it will assume already some familiarity. 

    

### 5.1. Create AWS instance


    bash src/bash/qto/qto.sh -a init-aws-instance
    
    # after that ssh to -it
    ssh -i ~/.ssh/id_rsa.prd.qto ubuntu@ec2-52-209-247-245.eu-west-1.compute.amazonaws.com

### 5.2. Setup bash & vim
This deployment script sets RATHER personal bash and tmux settings, which are NOT part of the QTO setup, but merely personal tools to navigate more easily in the terminal with bash, tmux and vim.

    curl https://raw.githubusercontent.com/YordanGeorgiev/ysg-confs/master/src/bash/deployer/setup-bash-n-vim.sh | bash -s <EMAIL>
    
    cat ~/

### 5.3. Clone project to Guest server
Connect via SSH to the target server and clone QTO Github repository as follows:

    cat ~/.ssh/
    
    mkdir -p ~/opt; cd $_ ;  git clone git@github.com:<<YourGitHubUserName>>/qto.git ; cd ~/opt

## 6. PHYSICAL HOST OS INSTALLATIONS


    

### 6.1. MacOs
QTO has been developed mostly by using MacOs as the physical host OS. The next code section is probably obsolete, as you most probably have installed it as described here: https://treehouse.github.io/installation-guides/mac/homebrew

    ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"

#### 6.1.1. Install QtPass
QtPass is a password manager. Installing it is optional.

https://sourceforge.net/p/gpgosx/docu/Download/

    brew cask install qtpass

## 7. POTENTIAL PROBLEMS AND TROUBLESHOOTING
As a rule of thumb, whatever errors you are encountering, they are most probably NOT source code errors, but configuration errors. Thus ALWAYS try to change configuration entries from the dev, tst or prd environment configuration files and restart the web server with the mojo-hypnotoad-start shell action.

    

### 7.1. Postgres admin user password is wrong
Some of the newly created tables might not have their permissions in the DDLs. Or you might have run some of the SQL scripts ad-hoc / manually and they do not contain grant statements.

The DB security is passed from OS root to Postgres user to QTO admin to QTO app, thus to fix it, run the following command, which will reprovision your Postgres and change DB user permissions.
You will also need to restart the web server after.

     ./src/bash/qto/qto.sh -a provision-db-admin -a run-qto-db-ddl -a load-db-data-from-s3 -a fix-db-permissions

### 7.2. AWS was not able to validate the provided access credentials
If that error occurs check your AWS credentials. Ensure that the AWS keys do not have the "+", "-" or "/" characters, as terraform does not support these symbols.

    data.aws_ami.ubuntu: Refreshing state...
    
    Error: AuthFailure: AWS was not able to validate the provided access credentials
            status code: 401, request id: 76ea7781-0bb5-4782-b358-f062c3de0a8b
    
      on main.tf line 8, in data "aws_ami" "ubuntu":
       8: data "aws_ami" "ubuntu" {

### 7.3. Redis refuses to start 
Redis should be configured to have bind on the IP address of the first eth. Also the bind to the ipv6 should be commented out ( ## bind 127.0.0.1 ::1). You should have also the correct IP configured in the cnf/env/&lt;&lt;env&gt;&gt;.env.json files. Because otherwise the application cannot connect and store correctly the DB metadata during application startup. There might be still some bugs during the Redis provisioning in the src/bash/deployer/check-install-redis.func.sh file, run the installation from the top to bottom command by command.

It may be worth checking redis.conf and make sure it is bound to a private IP, not a public one or localhost.

    sudo tail -n 10 /etc/redis/redis.conf
    
    # do not bind to ipv6 on the localhost 
    # bind 127.0.0.1 ::1
    
    # bind on the first eth0 ip address looking like 10.0.2.15 
    bind 10.0.62.82
    
    # enable service mode 
    supervised systemd
    
    ### -- Save redis.conf and return back to the Terminal
    
    # then restart Redis using this command in the terminal:
    sudo systemctl restart redis
    
    # you can verify that Redis is running in the following way:
    sudo systemctl status redis
    

### 7.4. Could not connect to Redis server
Uncaught exception from user code: Can't load application from file "...": Could not connect to Redis server at host-name:6379.
	
To fix this, first learn IPs of the current machine by running this command in the terminal:

ifconfig -a | grep inet

	
When working with virtual machines, search for the private IP that looks similar to 10.0.2.15.
When working with AWS ec2 instance, IP will look like 172.31.25.82.
	
Change Redis server IP inside %path-to-qto%/cnf/env/dev.env.json from host-name to the private IP. Then run mojo-hypnotoad again.
	
./src/bash/qto/qto.sh -a mojo-hypnotoad-start

	

    

### 7.5. You have reached the hw provisioning limits of your AWS account
If you get one of the errors below, go the UI of the AWS admin console and delete non-used resources. Fortunately, all resources have the &lt;&lt;env&gt;&gt;_&lt;&lt;resource_name&gt;&gt; naming convention either in the object or in their Tags, which means you will know what you are deleting.

    Error: Error creating VPC: VpcLimitExceeded: The maximum number of VPCs has been reached.
            status code: 400, request id: 06dbd26c-d7b2-46e1-853f-a9131c6ad9bd
    
      on main.tf line 41, in resource "aws_vpc" "tst-vpc":
      41: resource "aws_vpc" "tst-vpc" {
    
    # another one might be elastic ip's limits
    Error: Error creating EIP: AddressLimitExceeded: The maximum number of addresses has been reached.
            status code: 400, request id: 079d1264-8d2a-4dd3-9251-354634beda5c
    
      on main.tf line 203, in resource "aws_eip" "tst-ip-test":
     203: resource "aws_eip" "tst-ip-test" {

### 7.6. Cannot login in web interface with admin user
The password hashing in the users table is activated ALWAYS on blur even that the UI is not showing it (yes, that is more of a bug, than a feature).
The solution is to restart the application layer WITHOUT any authentication, change the admin user password from the UI and restart the application layer with authentication once again.

    export QTO_NO_AUTH=1
    bash src/bash/qto/qto.sh -a mojo-hypnotoad-start
    
    # now change the AdminEmail user password from the UI, delete the test users
    # as they all have the convenient "secret" password
    export QTO_NO_AUTH=0
    bash src/bash/qto/qto.sh -a mojo-hypnotoad-start
    

### 7.7. Mismatch in the AWS
The AWS web UI contains fancy Ajax calls and in our experience it does not always update properly, if are bombarding it with terraform deployments onto the same resources. Make sure you always hit F5 in your browser, when starting the work on new ec2 instance.

    

### 7.8. The problem occurred is not mentioned here ?!
Try to Google it first. If it took too long, send an e-mail / chat invitation to the instance owner you got the source code from and you WILL get help sooner or later.

    

<file_sep>#!/bin/bash
#------------------------------------------------------------------------------
# usage example:
# source $PRODUCT/lib/bash/funcs/export-json-section-vars.sh
# do_export_json_section_vars cnf/env/dev.env.json '.env.db'
#
# alias psql="PGPASSWORD=${postgres_sys_usr_admin_pw:-} psql -v -t -X -w -U \
#     ${postgres_sys_usr_admin:-} --port $postgres_rdbms_port --host $postgres_rdbms_host"
#------------------------------------------------------------------------------
do_export_json_section_vars(){

   json_file="$1"
   shift 1;
   test -f "$json_file" || echo "the json_file: $json_file does not exist !!! Nothing to do"
   test -f "$json_file" || exit 1

   section="$1"
   test -z "$section" && echo "the section in do_export_json_section_vars is empty !!! Nothing to do !!!"
   test -z "$section" && exit 1
   shift 1;

   echo "INFO exporting vars from cnf $json_file: "
   while read -r l ; do
      key=$(echo $l|cut -d'=' -f1|tr a-z A-Z)
      val=$(echo $l|cut -d'=' -f2)
      eval "$(echo -e 'export '$key=\"\"$val\"\")"
      echo "INFO $key=$val"
   done < <(cat "$json_file"| jq -r "$section"'|keys_unsorted[] as $key|"\($key)=\"\(.[$key])\""')
}
<file_sep># attempt to install terraform, if it is missing !!! Obs needs sudo !!!
doChkInstallTerraform(){

   which terraform  2>/dev/null || { 
      ver='0.12.8'
      wget -O "/tmp/terraform_""$ver'_linux_amd64.zip" \
         "https://releases.hashicorp.com/terraform/$ver/terraform_""$ver"'_linux_amd64.zip'
      sudo unzip -o "/tmp/terraform_""$ver'_linux_amd64.zip" -d '/usr/local/bin'
      terraform --version
   }

   which terraform  2>/dev/null || { 
      echo >&2 "The terraform binary is missing ! Aborting ..."; exit 1; }
   which terraform && echo "ok"
   
}
<file_sep>const vm = new Vue({
  el: '#app',
  data: {
    monthly_issues: []
  },
  mounted() {
    axios.get("/tst_qto/select/monthly_issues")
    .then(response => {this.monthly_issues = response.data.dat
	 })
  }
	
});
<file_sep>pyyaml
glob3
jinja2
pprintpp
pprintjson
requests
jq
<file_sep># file: src/bash/qto/funcs/generate-pdf-docs.func.sh

doGenerateMsftDocs(){
   test -z "${PROJ_INSTANCE_DIR-}" && PROJ_INSTANCE_DIR="$PRODUCT_DIR"
   test -z "${docs_root_dir-}" && docs_root_dir="$PROJ_INSTANCE_DIR"

   test -z ${PROJ_CONF_FILE-} && PROJ_CONF_FILE=$PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json
   do_export_json_section_vars $PROJ_CONF_FILE '.env.app'

   # <<web-host>>:<<web-port>>/<<db>>/select/export_files?as=grid&od=id
   basic_url="$ht_protocol://${web_host:-}:${port:-}/${postgres_app_db:-}"
   furl="$basic_url"'/select/export_files?as=grid&od=id&pg-size=20'
   echo "running: curl --cookie ~/.qto/cookies.txt --insecure  -s $furl \| jq -r '.dat[]|.url'"
   curl --cookie ~/.qto/cookies.txt --insecure  -s $furl | jq -r '.dat[]|.url'
   ret=$?
   test $ret != "0" && do_exit $ret "failed to get data from the $furl"

   while read -r url ; do 
      table_name=$(curl --cookie ~/.qto/cookies.txt --insecure  -s $furl | jq -r '.dat[]| select(.url=='\"$url\"')| .name'); 
      file_name="$table_name.docx"
      rel_path=$(curl --cookie ~/.qto/cookies.txt --insecure  -s $furl | jq -r '.dat[]| select(.url=='\"$url\"')| .path');
      [ "${rel_path-}" == "null" ] && rel_path="doc"
      rel_path="$rel_path/docx"
      mkdir -p $docs_root_dir/$rel_path
      file_path=$(echo $docs_root_dir/$rel_path/$file_name|perl -ne 's|[/]{2,5}|/|g;print')
      echo -e "\nrunning: python src/python/generate-docx.py $table_name"
      python3 src/python/generate-docx.py "$table_name" "$docs_root_dir/$rel_path"
   done < <(curl --cookie ~/.qto/cookies.txt --insecure  -s $furl | jq -r '.dat[]|.url')

   do_flush_screen
}

<file_sep>#!/bin/bash

# v0.8.8
# ---------------------------------------------------------
# call all the unit tests - fail if even one fails ...
# ---------------------------------------------------------
do_run_unit_tests(){

  export PATH=$PATH:~/perl5/bin/
  export PERL5LIB=${PERL5LIB:-}:~/perl5/lib/perl5/:$PRODUCT_DIR/src/perl/qto/t/lib:$PRODUCT_DIR/src/perl/qto/public/lib:$PRODUCT_DIR/src/perl/qto/lib

   export QTO_NO_AUTH=1
   test -z "${PROJ_INSTANCE_DIR-}" && PROJ_INSTANCE_DIR="$PRODUCT_DIR"
   #
   do_export_json_section_vars $PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json '.env.db'
   do_export_json_section_vars $PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json '.env.app'

   do_log "INFO START running the unit tests"
   while read -r f ; do
      do_log "INFO START unit test for $f"
      perl -MCarp::Always $f
      test $? -ne 0 && do_exit $? " the tests in the $f failed !!!"
      do_log "INFO STOP  unit test for $f"
      sleep 1
      do_flush_screen
   done < <(find src/perl/qto/t/lib/Qto/App -name '*.t' -type f |grep -v benchmarks |sort)

   export QTO_NO_AUTH=0
}

<file_sep># src/bash/qto/funcs/backup-postgres-db.func.sh

# ---------------------------------------------------------
# create a full database backup containing db create clause
# ---------------------------------------------------------
doBackupPostgresDb(){

   #
   test -z ${PROJ_CONF_FILE:-} && export PROJ_CONF_FILE="$PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json"

   do_export_json_section_vars $PROJ_CONF_FILE '.env.db'

   mix_data_dir="$PROJ_INSTANCE_DIR/dat/mix"

   backup_dir=$mix_data_dir/$(date "+%Y")/$(date "+%Y-%m")/$(date "+%Y-%m-%d")/sql/$postgres_app_db
   backup_fname=$postgres_app_db.`date "+%Y%m%d_%H%M%S"`.full.dmp.sql
   test -d $backup_dir || mkdir -p $backup_dir

   which nmap 2>/dev/null && nmap -Pnv -p $postgres_rdbms_port $postgres_rdbms_host || \
         do_exit 1 "cannot connect to $postgres_rdbms_port:$postgres_rdbms_host"

   PGPASSWORD="${postgres_sys_usr_admin_pw:-}" pg_dump -U "${postgres_sys_usr_admin:-}"  \
         -h $postgres_rdbms_host -p $postgres_rdbms_port -w --format=plain \
         $postgres_app_db   > $backup_dir/$backup_fname

   ls -1 $mix_data_dir/$(date "+%Y")/$(date "+%Y-%m")/$(date "+%Y-%m-%d")/sql/$postgres_app_db/* | sort -nr
   echo -e "\n"
   wc -l $mix_data_dir/$(date "+%Y")/$(date "+%Y-%m")/$(date "+%Y-%m-%d")/sql/$postgres_app_db/* | sort -nr
   sleep 1

   echo "Full database backup completed successfully."

}
# eof file: src/bash/qto/funcs/backup-postgres-db.func.sh
<file_sep>#!/bin/bash
#------------------------------------------------------------------------------
# Purpose:
# to search for a string and replace it with another recursively in a dir
# both in dir and file paths and their contents
# Prerequisites: setting vars in the caller shell
# export dir_to_morph=<<the-dir-to-search-and-replace-in-recursively>>
# export to_srch=<<the-string-to-search-for>>
# export to_repl=<<the-string-to-replace-with>>
# while read -r f ; do cp -v $f $(echo $f|perl -ne 's#func#help#g;print'|perl -ne
# 's#src#doc#g;print'|perl -ne 's#bash#txt#g;print'|perl -ne 's#help.sh#help.txt#g;print') ; done <
# <(find src/bash/run/ -type f)
#------------------------------------------------------------------------------
do_morph_dir(){

  # set -x
  # some initial checks the users should set the vars in their shells !!!
  test -z $dir_to_morph && {
    echo "You must export dir_to_morph=<<the-dir>> - it is empty !!!"; exit 1; }
  test -d $dir_to_morph || {
    echo "The dir to morph : \"$dir_to_morph\" is not a dir !!!"; exit 1; }
  test -z $to_srch && {
    echo "You must export to_srch=<<str-to-search-for>> - it is empty !!!"; exit 1; }
  test -z $to_repl && {
        echo "You must export to_repl=<<str-to-replace-with>> - it is empty !!!";exit 1; }

  do_log "INFO dir_to_morph: $dir_to_morph"
  do_log "INFO to_srch:\"$to_srch\" " ;
  do_log "INFO to_repl:\"$to_repl\" " ;
  sleep 2

  do_log "INFO START :: search and replace in non-binary files"
  #search and replace ONLY in the txt files and omit the binary files
  while read -r file ; do (
    #debug do_log doing find and replace in $file
    do_log "DEBUG working on file: $file"
    do_log "DEBUG searching for $to_srch , replacing with :: $to_repl"

        # we do not want to mess with out .git dir
        # or how-to check that a string contains another string
        case "$file" in
          *.git*)
          continue
          ;;
        esac
    perl -pi -e "s|\Q$to_srch\E|$to_repl|g" "$file"
  );
  done < <(find $dir_to_morph -type f -not -path "*/.venv/*" -not -exec file {} \; | grep text | cut -d: -f1)

  do_log "INFO STOP  :: search and replace in non-binary files"

  #search and repl %var_id% with var_id_val in deploy_tmp_dir
  do_log "INFO search and replace in dir and file paths dir_to_morph:$dir_to_morph"
    # rename the dirs according to the pattern
    while read -r dir ; do (
        echo $dir|perl -nle '$o=$_;s#'"\Q$to_srch\E"'#'"$to_repl"'#g;$n=$_;`mkdir -p $n` ;'
    );
    done < <(find $dir_to_morph -type d -not -path "*/.venv/*" \
        -not -path "/*node_modules/*" |grep -v '.git')

    # rename the files according to the pattern
    while read -r file ; do (
        echo $file | perl -nle '$o=$_;s|'"\Q$to_srch\E"'|'"$to_repl"'|g;$n=$_;rename($o,$n) unless -e $n ;'
    );
    done < <(find $dir_to_morph -type f -not -path "*/.venv/*" \
        -not -path "*/node_modules/*" |grep -v '.git')

    while read -r dir ; do (
        rm -rv $dir
    );
    done < <(find $dir_to_morph -type d -not -path "*/.venv/*" -not -path "/*node_modules/*" \
        | grep -v '.git'|grep "$to_srch")


    export exit_code=0

}

#eof file: src/bash/run/morph-dir.func.sh
<file_sep># src/bash/qto/funcs/increase-date.test.sh

doTestIncreaseDate() {
  set -eu
  sleep "$sleep_interval"
  run -a increase-date
  test $exit_code -ne 0 && return
}
<file_sep>-- DROP TABLE IF EXISTS projects ; 

SELECT 'create the "projects" table'
; 
   CREATE TABLE projects (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , name           varchar (100) NOT NULL DEFAULT 'name...'
    , description    varchar (4000)
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_projects_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

create unique index idx_uniq_projects_id on projects (id);

-- the rank search index
CREATE INDEX idx_rank_projects ON projects
USING gin(to_tsvector('English', name || ' ' || description || 'owner')); 


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.projects'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 



--The trigger:
CREATE TRIGGER trg_set_update_time_on_projects BEFORE UPDATE ON projects FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'projects'::regclass;
<file_sep># src/bash/qto/funcs/run-integration-tests.test.sh

# v1.3.0
doTestRunIntegrationTests(){

	do_log "DEBUG START doTestRunIntegrationTests"
	
	# Action !!!
   bash src/bash/qto/qto.sh -a run-integration-tests

	do_log "DEBUG STOP  doTestRunIntegrationTests"
}
# eof func doTestRunIntegrationTests


# eof file: src/bash/qto/funcs/run-integration-tests.test.sh
<file_sep># file: src/bash/qto/funcs/remove-all-docker-images.sh

# v0.6.5
# ---------------------------------------------------------
# remove all the docker images on the host with warning
# and inform if none exists
#$ docker system prune -a --volumes
#$ docker image prune
#$ docker container prune
#$ docker volume prune
#$ docker network prune
# ---------------------------------------------------------
doRemoveDockerImages(){
   do_flush_screen
   do_log "INFO START removing all docker images !!! " 
   do_log "INFO Are you SURE ??!! You have 3 seconds to abort by Ctrl + C !!"
   sleep 3
  
   ret=$(docker images|grep qto-image|grep $product_version.$ENV|awk '{print $3}'|wc -l|perl -ne 's/\s+//g;print')
   if test $ret -eq 0 
   then
      do_log "INFO No docker images found. Nothing to do !!!"
   else
      docker rmi $(docker images | grep qto-image | grep $product_version.$ENV |awk '{print $3}')
   fi

   do_log "INFO STOP removing all the docker images"
}


# eof file: src/bash/qto/funcs/remove-all-docker-images.sh
<file_sep># src/bash/qto/funcs/gmail-package.help.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doHelpGmailPackage comments ...
# ---------------------------------------------------------
doHelpGmailPackage(){

	do_log "DEBUG START doHelpGmailPackage"
	
	cat doc/txt/qto/helps/gmail-package.help.txt
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!

	do_log "DEBUG STOP  doHelpGmailPackage"
}
# eof func doHelpGmailPackage


# eof file: src/bash/qto/funcs/gmail-package.help.sh
<file_sep>#!/bin/bash

do_restart_network(){

	test -z ${USER:-} && export USER=$(echo $HOME|cut -d'/' -f 3)
	test `sudo -n -l -U $USER 2>&1 | egrep -c -i "not allowed to run sudo|unknown user"` -eq 0 \
		|| do_log "FATAL you need to have sudo to run this script !!!"

   sudo /etc/init.d/networking restart

   do_log "INFO start restarting postgres ..."
   sudo sh /etc/init.d/postgresql restart
   do_log "INFO stop  restarting postgres ..."
}
<file_sep># src/bash/qto/funcs/run-functional-tests.help.sh

# v1.0.9
# ---------------------------------------------------------
# cat doc/txt/qto/helps/perl/run-functional-tests.help.txt
# ---------------------------------------------------------
doHelpRunFunctionalTests(){

	do_log "DEBUG START doHelpRunFunctionalTests"
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 

	do_log "DEBUG STOP  doHelpRunFunctionalTests"
}
# eof func doHelpRunFunctionalTests


# eof file: src/bash/qto/funcs/run-functional-tests.help.sh
<file_sep>-- file: src/sql/pgsql/list-postgres-schemas.sql
-- usage: 
/**
alias psql="PGPASSWORD=${postgres_app_usr_pw:-} psql -v -q -t -X -w -U ${postgres_app_usr:-}"
alias psql="PGPASSWORD=${postgres_sys_usr_admin_pw:-} psql -v -q -t -X -w -U ${postgres_sys_usr_admin:-}"
psql -d dev_qto < src/sql/pgsql/list-postgres-schemas.sql | less
*/

select schema_name
from information_schema.schemata;

-- eof file: src/sql/pgsql/list-postgres-schemas.sql
<file_sep># src/bash/qto/funcs/log-test-run-entry.func.sh

# 
# ---------------------------------------------------------
# provides a mean for registering test run log entries 
# Usage:
# do_logTestRunEntry START "$msg"
# do_logTestRunEntry INFO "$msg"
# do_logTestRunEntry STOP $exit_code "$msg"
# do_logTestRunEntry STATUS
# ---------------------------------------------------------
do_logTestRunEntry(){

	do_log "DEBUG START do_logTestRunEntry"

   case "$1" in
     INIT)
         # cat doc/txt/qto/funcs/log-test-run-entry.func.txt
         component_name=$RUN_UNIT
         test -z "$component_name" && component_name="$RUN_UNIT_tester"
         test -z "$test_run_report_line" && test_run_report_line='   '
         test -z "$test_run_report_file" \
         && test_run_report_file="$PRODUCT_DIR/dat/tests/$component_name"'.test-run-report.'`date "+%Y%m%d_%H%M%S"`'.txt'
         echo -e "\n" > "$test_run_report_file"
         echo -e `date "+%Y-%m-%d %H:%M:%S"`"\t START $component_name test run report \n" >> "$test_run_report_file"
         echo "result  start-time  stop-time   action-name" >> "$test_run_report_file"
         shift;
         ;;
     START)
         test_run_report_line='   nok   '
         shift;
         ;;
     INFO)
         shift;
         test_run_report_line="$test_run_report_line""$*"
         ;;
     STOP)
         shift;
         exit_code=$1
         if test "$exit_code" -eq 0; then
            test_run_report_line=$(echo "$test_run_report_line"|perl -ne 's/   nok   /   ok   /g;print')
         fi
         echo -e "$test_run_report_line" >> "$test_run_report_file"
         ;;
     STATUS)
         echo -e "\n\n"
         echo -e "\n\n" >> "$test_run_report_file"
         echo -e `date "+%Y-%m-%d %H:%M:%S"`"\t STOP  $component_name test run report" >> "$test_run_report_file"
         cat "$test_run_report_file"
         echo -e "\n\n"
         # do_log "product instance tests listing"
         # find "$PRODUCT_DIR"'/dat/tests' -type f -exec stat -c "%y %n" {} \; | sort -nr
         echo -e "\n\n"
         ;;
     *)
         echo $"Usage: $0 {START|STOP|INFO|STATUS}" 1>&2 
   esac	
        
	do_log "DEBUG STOP  do_logTestRunEntry"
}
# eof func do_logTestRunEntry


# eof file: src/bash/qto/funcs/log-test-run-entry.func.sh
<file_sep>#!/usr/bin/env bash

# how RATIONAL is to add code which introduces regression bugs !?
clear ; ./src/bash/qto/qto.sh -a run-unit-tests
clear ; ./src/bash/qto/qto.sh -a run-functional-tests

# scramble the secrets into random hashes
clear ; ./src/bash/qto/qto.sh -a scramble-confs
git update-index --assume-unchanged cnf/env/dev.env.json
git update-index --assume-unchanged cnf/env/tst.env.json
git update-index --assume-unchanged cnf/env/prd.env.json


# update the meta include file for each env from the dev env
for env in `echo dev tst prd` ; do cp -v met/.dev.qto met/.$env.qto ; done ;
<file_sep>#!/usr/bin/env bash

main(){
   do_set_vars "$@"  # is inside, unless --help flag is present
   ts=$(date "+%Y%m%d_%H%M%S")
   main_log_dir=~/var/log/$RUN_UNIT/; mkdir -p $main_log_dir
   main_exec "$@" \
    > >(tee $main_log_dir/$RUN_UNIT.$ts.out.log) \
    2> >(tee $main_log_dir/$RUN_UNIT.$ts.err.log)
}

main_exec(){
   do_resolve_os
   do_check_install_min_req_bins
   do_load_functions

   test -z ${actions:-} || {
      do_run_actions "$actions"
   }
   do_finalize
}


#------------------------------------------------------------------------------
# the "reflection" func - identify the the funcs per file
#------------------------------------------------------------------------------
get_function_list () {
   env -i PATH=/bin:/usr/bin:/usr/local/bin bash --noprofile --norc -c '
      source "'"$1"'"
      typeset -f |
      grep '\''^[^{} ].* () $'\'' |
      awk "{print \$1}" |
      while read -r fnc_name; do
         type "$fnc_name" | head -n 1 | grep -q "is a function$" || continue
            echo "$fnc_name"
            done
            '
}


do_read_cmd_args() {

   while [[ $# -gt 0 ]]; do
      case "$1" in
         -a|--actions) shift && actions="${actions:-}${1:-} " && shift ;;
         -h|--help) actions=' do_print_usage ' && ENV=lde && shift ;;
         *) echo FATAL unknown "cmd arg: '$1' - invalid cmd arg, probably a typo !!!" && shift && exit 1
    esac
  done
   shift $((OPTIND -1))

}


do_run_actions(){
   actions=$1
      cd $PRODUCT_DIR
      actions="$(echo -e "${actions}"|sed -e 's/^[[:space:]]*//')"  #or how-to trim leading space
      run_funcs=''
      while read -d ' ' arg_action ; do
         while read -r fnc_file ; do
            #debug func fnc_file:$fnc_file
            while read -r fnc_name ; do
               #debug fnc_name:$fnc_name
               action_name=`echo $(basename $fnc_file)|sed -e 's/.func.sh//g'`
               action_name=`echo do_$action_name|sed -e 's/-/_/g'`
               # debug  action_name: $action_name
               test "$action_name" != "$arg_action" && continue
               source $fnc_file
               test "$action_name" == "$arg_action" && run_funcs="$(echo -e "${run_funcs}\n$fnc_name")"
            done< <(get_function_list "$fnc_file")
         done < <(find "src/bash/run/" -type f -name '*.func.sh'|sort)

      done < <(echo "$actions")

   run_funcs="$(echo -e "${run_funcs}"|sed -e 's/^[[:space:]]*//;/^$/d')"
   while read -r run_func ; do
      cd $PRODUCT_DIR
      do_log "INFO START ::: running action :: $run_func"
      $run_func
      if [[ "${exit_code:-}" != "0" ]]; then
        do_log "FATAL failed to run action: $run_func !!!"
         exit $exit_code
      fi
      do_log "INFO STOP ::: running function :: $run_func"
   done < <(echo "$run_funcs")

}


do_flush_screen(){
   # printf "\033[2J";printf "\033[0;0H"
   echo ""
}


#------------------------------------------------------------------------------
# echo pass params and print them to a log file and terminal
# usage:
# do_log "INFO some info message"
# do_log "DEBUG some debug message"
#------------------------------------------------------------------------------
do_log(){
   type_of_msg=$(echo $*|cut -d" " -f1)
   msg="$(echo $*|cut -d" " -f2-)"
   log_dir="${PRODUCT_DIR:-}/dat/log/bash" ; mkdir -p $log_dir
   log_file="$log_dir/${RUN_UNIT:-}.`date "+%Y%m%d"`.log"
   echo " [$type_of_msg] `date "+%Y-%m-%d %H:%M:%S %Z"` [${RUN_UNIT:-}][@${host_name:-}] [$$] $msg " | \
      tee -a $log_file
}


do_check_install_min_req_bins(){
   which perl > /dev/null 2>&1 || {
      run_os_func install_bins perl
   }
   which jq > /dev/null 2>&1 || {
      run_os_func install_bins jq
   }
   which make > /dev/null 2>&1 || {
      # this will not work properly - google how-to install make on <<my-operating-system>>
      run_os_func install_bins make
   }
}


do_set_vars(){
   set -u -o pipefail
   do_read_cmd_args "$@"
   # test $? -eq "0" && export exit_code='0'
   export exit_code=1 # assume failure for each action, enforce return code usage
   export host_name="$(hostname -s)"
   unit_run_dir=$(perl -e 'use File::Basename; use Cwd "abs_path"; print dirname(abs_path(@ARGV[0]));' -- "$0")
   export RUN_UNIT=$(cd $unit_run_dir/../../.. ; basename `pwd`)
   export PRODUCT_DIR=$(cd $unit_run_dir/../../.. ; echo `pwd`)
   ENV="${ENV:=lde}" # https://stackoverflow.com/a/2013589/65706
   valid_envs=("lde dev prd stg all")
   [[ " ${valid_envs[*]} " =~ " ${ENV} " ]] || {
      echo -e "ENV must be one of the following : \n export ENV={lde dev stg prd all}" && exit 1
   }

   cd $PRODUCT_DIR
   # workaround for github actions running on docker
   test -z ${GROUP:-} && export GROUP=$(id -gn)
   test -z ${GROUP:-} && export GROUP=$(ps -o group,supgrp $$|tail -n 1|awk '{print $1}')
   test -z ${USER:-} && export USER=$(id -un)
   test -z ${UID:-} && export UID=$(id -u)
   test -z ${GID:-} && export GID=$(id -g)
}


do_finalize(){

   # do_flush_screen

   cat << EOF_FIN_MSG
   :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
         $RUN_UNIT run completed
   :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
EOF_FIN_MSG
}


do_load_functions(){
    while read -r f; do source $f; done < <(ls -1 $PRODUCT_DIR/lib/bash/funcs/*.sh)
    while read -r f; do source $f; done < <(ls -1 $PRODUCT_DIR/src/bash/run/*.func.sh)
 }

run_os_func(){
   func_to_run=$1 ; shift ;

   if [ $(uname -s) == *"Linux"* ]; then
       distro=$(cat /etc/os-release|egrep '^ID='|cut -d= -f2)
       if [ $distro == "ubuntu" ] || [ $distro == "pop" ]; then
      "do_ubuntu_""$func_to_run" "$@"
       elif [ $distro == "alpine" ]; then
      "do_alpine_""$func_to_run" "$@"
       else
          echo "your Linux distro is not supported !!!"
       fi
   elif [ $(uname -s) == *"Linux"* ]; then
      echo "you are running on mac"
      "do_mac_""$func_to_run" "$@"
   fi

}


do_resolve_os(){
   if [ $(uname -s) == *"Linux"* ]; then
       distro=$(cat /etc/os-release|egrep '^ID='|cut -d= -f2)
       if [ $distro == "ubuntu" ] || [ $distro == "pop" ]; then
         export OS=ubuntu
       elif [ $distro == "alpine" ]; then
         export OS=alpine
       else
          echo "your Linux distro is not supported !!!"
       fi
   elif [ $(uname -s) == *"Darwin"* ]; then
         export OS=mac
   fi

}

main "$@"
<file_sep>#!/bin/bash

do_alpine_chk_install_perl_modules(){
   cd $PRODUCT_DIR
   modules="$(cat bin/cnf/perl/qto-app-perl-modules.lst)"

   while read -r module ; do
      use_modules="${use_modules:-} use $module ; "
   done < <(echo "$modules");

   set -x
   perl -e "$use_modules" || {
      echo "deploying modules. This WILL take couple of min, but ONLY ONCE !!!"
      curl -L http://cpanmin.us | sudo perl - --self-upgrade -l ~/perl5 App::cpanminus \
         && sudo chown -R $APPUSR:$APPGRP ~/.cpanm && sudo chown -R $APPUSR:$APPGRP ~/perl5
      ~/perl5/bin/cpanm --local-lib=~/perl5 local::lib && eval $(perl -I ~/perl5/lib/perl5/ -Mlocal::lib)
      curl -L cpanmin.us | perl - Mojolicious
      test "$(grep -c 'Mlocal::lib' ~/.bashrc|xargs)" -eq 0 && \
         echo 'eval $(perl -I$HOME/perl5/lib/perl5 -Mlocal::lib)' >> ~/.bashrc
    while read -r module ; do cpanm_modules="${cpanm_modules:-} $module " ; done < <(echo "$modules")

  #cpanm --force --local-lib=~/perl5 'Net::Google::DataAPI' # does not install otherwise !!!
    # probably not needed as all google related modules are out of the setup ^^^
  cmd="cpanm --local-lib=~/perl5 $modules"
  $cmd

  # nasty workaround for the "Moo complainings" in the Net::Google::DataAPI::Oauth2 module
  find ~ -type f -name OAuth2.pm -exec perl -pi -e \
    "s/use Any::Moose;/no warnings 'deprecated'; use Any::Moose; use warnings 'deprecated';/g" {} \;
  }

  set +x
  # create the symlink to the hypnotoad
  export tgt_path=$HOME/perl5/bin/hypnotoad
  export link_path=/usr/local/sbin/hypnotoad
  sudo mkdir -p `dirname $link_path`
  sudo test -L $link_path && unlink $link_path
  sudo ln -s $tgt_path $link_path
  ls -la $link_path;


  export exit_code=0
}
<file_sep>// src: https://stackoverflow.com/a/47181562/65706
Vue.component('editable', {
  template: `
    <div v-bind:id="id" contenteditable="true" @blur="emitChange">
      {{ content }}
    </div>
  `,
  props: ['content','id'],
  methods: {
    emitChange (ev) {
      this.$emit('update', ev.target.textContent,ev.target.id)
    }
  }
})

new Vue({
  el: '#app',
  data: {
    herobanner: {
      headline: 'Parent is updated on blur event, so click outside this text to update it.'
		, id: "12"
    }
  },
  methods: {
    async loadJson () {
      var response = await fetch('https://swapi.co/api/people/1')
      var hero = await response.json()
      this.herobanner.headline = hero.name
    },
    updateHeadline (content,id) {
      this.herobanner.headline = content
		console.log ( "new content: " + content ) 
		console.log ( "with id: " + id )
    }
  }
})
<file_sep>/** usage: 
	clear ; cd src/js/node/pp-ui-tests ; npm test ; cd - ;
*/
'use strict';
const puppeteer = require('puppeteer');
const path = require('path')

async function main(){
	const browser = await puppeteer.launch({headless: false});
	try {
		var ConfFile = getProductInstanceDir() + "cnf/env/dev.env.json";
		var json = require(ConfFile);
		const page = await browser.newPage();
		await page.setViewport({width: 1200, height: 720})
		await testLogin(json.env, page);
		await testViewDoc(json.env,page)
	} catch (error) {
	  console.log(error);
	} finally {
	  await browser.close();
	}
}

async function testLogin(env,page) {
		var url = env.app.ht_protocol + '://' + env.app.web_host + ':' + env.app.mojo_morbo_port + '/qto/login';
		await page.goto(url, { waitUntil: 'networkidle0' }); // wait until
		await page.type('#email', '<EMAIL>');
		//await page.type('#pass', '<PASSWORD>');
		await page.waitFor(1000);
		await Promise.all([
			page.click('#butLogin'),
			page.waitForNavigation({ waitUntil: 'networkidle0' }),
		]);
}


async function testViewDoc(env,page) {
		var uri = env.app.ht_protocol + '://' + env.app.web_host + ':' + env.app.mojo_morbo_port + '/qto/view/test_hierarchy_doc';
		await page.goto(uri, { waitUntil: 'networkidle0' }); 
		//await page.type('#inp_srch_box', 'name-01'); //test quick search
		await page.waitFor(1000);
		page.click('#div_open_rgt_menu'); //test thie open right menu action
		await page.waitFor(1000);
		await page.$$eval('a.cls-context-menu-link', elHandles => elHandles.forEach(el => el.click()))
		//nopeawait page.$$eval('a.cls-context-menu-link', elHandles => elHandles.forEach(el => el.click({button: 'right',delay: 1000,})))

		// todo:right click the test_hierarchy_doc-8
		page.click('#div_open_rgt_menu'); //test the close right menu action
		page.waitFor(1000);
		await page.waitFor(3000);
}


function getProductInstanceDir(){
	return path.join(path.dirname(__filename), '../../../../../'); 
	
}

describe('view doc page tests', function () {
  it(`should 
			- open the login page, login
			- open the view-doc page, 
			- open the right menu
			- click on all the num order links
			- close the right menu`
			, async function () {
      main();
  });
});

function sleep(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
}
<file_sep>escape(){
        echo "$1"|perl -MURI::Escape -ne 'chomp;print uri_escape($_),"\n"'
}
unescape(){
        echo "$1"|perl -MURI::Escape -ne 'chomp;print uri_unescape($_),"\n"'
}
encrypt_pass(){
 echo "$1"| perl -ne '$_=crypt($_,$_); s/[#\-%&\$*+()\/]//g; print substr (sprintf("%s%s", $_ ,"56!"), 8, 10, "")'
}
<file_sep># src/bash/qto/funcs/build-docker-image.test.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doTestBuildQtoDockerImage comments ...
# cat doc/txt/qto/tests/build-docker-image.test.txt
# ---------------------------------------------------------
doTestBuildQtoDockerImage(){

	do_log "DEBUG START doTestBuildQtoDockerImage"
	
	# Action !!!
   bash "$PRODUCT_DIR/src/bash/qto/qto.sh" -a build-docker-image
	do_log "DEBUG STOP  doTestBuildQtoDockerImage"
}
# eof func doTestBuildQtoDockerImage


# eof file: src/bash/qto/funcs/build-docker-image.test.sh
<file_sep>#!/usr/bin/env bash

main(){

   do_set_vars "$@"  # is inside, unless --help flag is present
   
   deploy_step_funcs=(
      do_load_functions
#      do_chk_sudo_rights
#      do_chk_setup_bash
#      do_chk_install_os_packages
#      do_chk_install_postgres
#      do_chk_install_perl_modules
#      do_chk_install_python_modules
#     do_chk_install_redis
      do_scramble_confs
#     do_chk_provision_postgres   
#      do_chk_provision_db_users
#      do_create_app_db
#      do_run_ddls
#      do_create_app_user
#      do_configure_aws_keys
#      do_create_ssh_keys
#      do_chk_install_python_modules
#      do_chk_install_nginx
#      do_chk_provision_nginx
#      do_add_dns
#      do_change_file_permissions
#      do_sync_time
      # do_chk_install_terraform
      # do_chk_install_phantom_js
      do_finalize
   )

   counter=0;
   for i in ${!deploy_step_funcs[*]}
   do
	  ((counter+=1))
     run_step=`echo ${deploy_step_funcs[$i]}|cut -c 4-`
     do_log "INFO START $run_step"
     printf "$counter/${#deploy_step_funcs[*]} $run_step \n\n";
     ${deploy_step_funcs[$i]}
     do_log "INFO STOP $run_step"
     sleep 1 ; do_flush_screen
   done
}

do_flush_screen(){
   printf "\033[2J";printf "\033[0;0H"
}

#------------------------------------------------------------------------------
# echo pass params and print them to a log file and terminal
# usage:
# do_log "INFO some info message"
# do_log "DEBUG some debug message"
#------------------------------------------------------------------------------
do_log(){
   type_of_msg=$(echo $*|cut -d" " -f1)
   msg="$(echo $*|cut -d" " -f2-)"
   [[ -t 1 ]] && echo " [$type_of_msg] `date "+%Y-%m-%d %H:%M:%S %Z"` [$run_unit][@$host_name] [$$] $msg "
   log_dir="$PRODUCT_DIR/dat/log/bash" ; mkdir -p $log_dir && log_file="$log_dir/$run_unit.`date "+%Y%m"`.log"
   printf " [$type_of_msg] `date "+%Y-%m-%d %H:%M:%S %Z"` [$run_unit][@$host_name] [$$] $msg " >> $log_file
}


do_set_vars(){
   set -eu -o pipefail
   export run_unit=${1:-qto}
   maybe_echo=${2:-} || ''
   export host_name="$(hostname -s)"
   unit_run_dir=$(perl -e 'use File::Basename; use Cwd "abs_path"; print dirname(abs_path(@ARGV[0]));' -- "$0")
   export PRODUCT_DIR=$(cd $unit_run_dir/../../../../..; echo `pwd`)
   bash_opts_file=~/.bash_opts.`hostname -s`
   cd $PRODUCT_DIR
}


usage_info(){
   # if $run_unit is --help, then message will be "--help deployer PURPOSE"
   cat << EOF_USAGE
   :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
      qto deployer PURPOSE: 
      to create the qto backend base image from an Ubuntu 18.04 with 
      a single shell call by deploying and configuring : 
       - OS packages and utils
       - Perl and Python modules
       - NodeJS runtime and npm modules
       - Postgres db enginge
   :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
     
EOF_USAGE
   do_exit 0 'usage displayed'
}


do_chk_sudo_rights(){
   printf "\nChecking sudo rights.\n\n"
   msg='is not allowed to run sudo'
   test $(sudo -l -U $USER 2>&1 | grep -c "$msg") -eq 1 && echo "$USER $msg !!!" && exit 1
   printf "OK\n"
}


do_sync_time(){
   printf "\nSynchronising time.\n\n"
   # note ntpdate is deprecated : https://askubuntu.com/a/998449/251228
   sudo apt-get install -y ntp
   sudo timedatectl set-ntp on
   sudo service ntp restart
   sudo service ntp status | cat
}


do_add_dns(){
	if [[ ! "grep -q 'nameserver 8.8.8.8' /etc/resolv.conf" ]] ;  # skip adding, if resolve.conf already contains 8.8.8.8
	then
		printf "\nAdding DNS to /etc/resolv.conf\n\n"
		set -x
      cat << EOF_ADD_DNS | sudo tee -a /etc/resolv.conf > /dev/null
nameserver 8.8.8.8
nameserver 8.8.4.4
EOF_ADD_DNS
	fi
}


do_load_functions(){
   while read -r f; do source $f; done < <(ls -1 $PRODUCT_DIR/lib/bash/funcs/*.sh)
   export -f do_flush_screen
   export -f do_export_json_section_vars
   while read -r f; do source $f; done < <(ls -1 $PRODUCT_DIR/src/bash/qto/deploy/single-ubuntu-18.04/*func.sh)
}




do_change_file_permissions(){
   find $PRODUCT_DIR -type f -name '*.sh' -exec chmod 770 {} \;
}


do_finalize(){
   set +x
   
   do_flush_screen
   
   cat << EOF_INIT_MSG
   :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
         qto binary deployment completed successfully. Continue by : 
   :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
EOF_INIT_MSG
}



# ------------------------------------------------------------------------------
# clean and exit with passed status and message
# call by:
# do_exit 0 "ok msg"
# do_exit 1 "NOK msg"
# ------------------------------------------------------------------------------
do_exit(){
   exit_code=$1 ; shift
   exit_msg="$*"

   if (( ${exit_code:-} != 0 ));
   then
      exit_msg=" ERROR --- exit_code $exit_code --- exit_msg : $exit_msg"
      >&2 printf "$exit_msg"
      do_log "FATAL STOP FOR $run_unit deployer RUN with: "
      do_log "FATAL exit_code: $exit_code exit_msg: $exit_msg"
   else
      do_log "INFO STOP FOR $run_unit deployer RUN with: "
      do_log "INFO STOP FOR $run_unit deployer RUN: $exit_code $exit_msg"
   fi

   exit $exit_code
}


main "$@"
<file_sep>---- DROP TABLE IF EXISTS export_files ; 

SELECT 'create the "export_files" table'
; 
   CREATE TABLE export_files (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , url            varchar (200) NOT NULL DEFAULT 'url...'
    , path           varchar (50) NULL DEFAULT 'rel-path-from-doc-root' 
    , name           varchar (100) NOT NULL DEFAULT 'the-export-file-name'
    , description    varchar (200) NOT NULL DEFAULT 'description...'
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_export_files_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

create unique index idx_export_files_uniq_id on export_files (id);


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.export_files'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 

--The trigger:
CREATE TRIGGER trg_set_update_time_on_export_files BEFORE UPDATE ON export_files FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'export_files'::regclass;
<file_sep># file: src/bash/qto/funcs/full-clean-docker.sh

# v0.6.5
# ---------------------------------------------------------
# clean totally the docker runtime
# ---------------------------------------------------------
doFullCleanDocker(){
   do_flush_screen
   do_log "INFO START full clean of docker !!! " 
   do_log "INFO Are you SURE ??!! You have 3 seconds to abort by Ctrl + C !!"
   sleep 3
   
   docker system prune -f -a --volumes
   docker image prune -f
   docker container prune -f
   docker volume prune -f
   docker network prune -f

   do_log "INFO STOP  full clean of the docker"
}


# eof file: src/bash/qto/funcs/full-clean-docker.sh
<file_sep>
do_chk_install_ubuntu_packages(){
   
   packages_list_file="$product_dir/src/bash/deployer/$app_to_deploy/cnf/bin/ubuntu-18.04.lst"
   err_msg="the packages list file: $packages_list_file does not exist !"
   test -f $packages_list_file || do_exit 1 "$err_msg"
   packages="$(cat $packages_list_file)"

   sudo apt-get update      
	# sudo apt-get upgrade # probably not a good idea, but if yes ... this is the place ...
   while read -r package ; do 
      test "$(sudo dpkg -s $package | grep Status)" == "Status: install ok installed" || {
         $maybe_echo sudo apt-get install -y $package 
      }
   done < <(echo "$packages");
      
}
<file_sep>doRestartPostgres(){

	test -z ${USER:-} && export USER=$(echo $HOME|cut -d'/' -f 3)
	test `sudo -n -l -U $USER 2>&1 | egrep -c -i "not allowed to run sudo|unknown user"` -eq 0 \
		|| do_log "FATAL you need to have sudo to run this script !!!"

  
   # to prevent the 'failed: could not create socket: Too many open files at sys' error
   # perl -e 'while(1){open($a{$b++}, "<" ,"/dev/null") or die $b;print " $b"}' to check
   ulimit -Sn 4096

   do_log "INFO start restarting postgres ..."
   sudo sh /etc/init.d/postgresql restart
   do_log "INFO stop  restarting postgres ..."

}
<file_sep># src/make/install-dockers.func.mk
# only the clean install dockers calls here ...
# TODO: figure a more elegant and generic way to avoid this copy paste ...
#

SHELL = bash
PRODUCT := $(shell basename $$PWD)
ORG := $(shell export ORG=$${ORG}; echo $${ORG})


.PHONY: clean-install-app  ## @-> setup the whole local app environment no cache!
clean-install-app:
	$(call install-img,app,8080,--no-cache)


.PHONY: install-app  ## @-> setup the whole local app environment no cache!
install-app:
	$(call install-img,app)


<file_sep># file: src/bash/qto/funcs/dev/create-ctags.func.sh

# v0.8.5
#------------------------------------------------------------------------------
# creates the ctags file for the projet
#------------------------------------------------------------------------------
doCreateCtags(){

	ctags --help >/dev/null 2>&1 ||
	   { do_log "ERROR. ctags is not installed or not in PATH. Aborting." >&2; exit 1; }
	pushd .
	cd $PRODUCT_DIR

	cmd="rm -fv ./tags"                          && doRunCmdAndLog "$cmd"
	cmd="ctags  -R -n --fields=+i+K+S+l+m+a --exclude=.git --exclude=dat --exclude=*/node_modules/* ."   && doRunCmdAndLog "$cmd"
	cmd="ls -la $PRODUCT_DIR/tags"       && doRunCmdAndLog "$cmd"

	popd

}
<file_sep>#!/bin/bash
# src/bash/qto/funcs/backup-postgres-table.func.sh

do_backup_postgres_table(){

   test -z ${PROJ_CONF_FILE:-} && export PROJ_CONF_FILE="$PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json"

   do_export_json_section_vars $PROJ_CONF_FILE '.env.db'
   pgsql_scripts_dir="$PROJ_INSTANCE_DIR/src/sql/pgsql/ad-hoc"
   sleep 3 ; do_flush_screen ;

   test -z ${table:-} && do_exit 1 "no table defined to backup !!!"
	table="$(echo -e "${table}" | sed -e 's/[[:space:]]*$//')" #or how-to to trin leading space
   dump_dir=$PROJ_INSTANCE_DIR/dat/mix/$(date "+%Y")/$(date "+%Y-%m")/$(date "+%Y-%m-%d")/tmp
   mkdir -p $dump_dir || do_exit 1 "cannot create the dump_dir: $dump_dir"
   dump_file=$dump_dir/${postgres_app_db:-}.${table:-}.$(date "+%Y%m%d_%H%M%S").data.sql

   echo "start running : pg_dump  --verbose --data-only --table public.$table $postgres_app_db "
   echo -e "\n --file $dump_file \n"

   # --verbose --data-only --table public.$table $postgres_app_db \
   PGPASSWORD="${postgres_sys_usr_admin_pw:-}" pg_dump -U "${postgres_sys_usr_admin:-}"  \
      -h $postgres_rdbms_host -p $postgres_rdbms_port -w \
      --verbose --table public.$table $postgres_app_db \
      --file $dump_file

   perl -pi -e "print \"DROP TABLE public.$table; \n \" if $. == 1" $dump_file
   echo "stop running : pg_dump  --verbose --data-only --table public.$table $postgres_app_db \\"
   echo -e " --file $dump_file \n"

   do_log "INFO produced the following file : $(ls -1 $dump_file)"
   do_log "INFO with the following amount of lines :"
   wc -l $dump_file
   echo -e "\n\n"
}

<file_sep>#!/bin/bash
# file: src/bash/run/check-perl-syntax.func.sh

do_check_perl_syntax() {

  export PATH=$PATH:~/perl5/bin/
  export PERL5LIB=${PERL5LIB:-}:~/perl5/lib/perl5/:$PRODUCT_DIR/src/perl/qto/t/lib:$PRODUCT_DIR/src/perl/qto/public/lib:$PRODUCT_DIR/src/perl/qto/lib

  find . -name autosplit.ix | xargs rm -fv # because idempotence
  declare -a rv
  rv=0

  # foreach perl file check the syntax by setting the correct INC dirs
  while read -r dir; do

    echo -e "\n start compiling $dir ..."
    cd $PRODUCT_DIR/src/perl/$dir

    # run the autoloader utility
    find . -name '*.pm' -exec perl -MAutoSplit -e 'autosplit($ARGV[0], $ARGV[1], 0, 1, 1)' {} \;

    c=0
    # feel free to adjust to 5, more might get you the "Out of memory!" error
    amount_of_perl_syntax_checks_to_run_in_parallel=1
    while read -r file; do
      c=$((c + 1))
      test $c -eq $amount_of_perl_syntax_checks_to_run_in_parallel && sleep 1 && export c=0
      (
        echo -e "\t ::: running: cd src/perl/qto ; perl -MCarp::Always -I $(pwd) -I $(pwd)/lib -wc \"$file\" ; cd -"
        perl -MCarp::Always -I $(pwd) -I $(pwd)/lib -wc "$file"
        # rv=$? ;  # probably not needed ... better just to spit out the error for the
        # test $rv -ne 0 && break 2 ;
      ) &
    done < <(find "." -type f \( -name "*.pl" -or -name "*.pm" -or -name '*.t' \))
    test $rv -ne 0 && break

    echo -e "stop compiling $dir ... \n\n"
    cd $PRODUCT_DIR || exit 1

  done < <(ls -1 "src/perl")

  test $rv -ne 0 && do_exit 4 "Perl syntax error"

  do_flush_screen
  export exit_code=0
}
<file_sep> ---- DROP TABLE IF EXISTS test_empty_table ; 

SELECT 'create the "test_empty_table" table'
;

   CREATE TABLE test_empty_table (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , name           varchar (200) NOT NULL DEFAULT 'name...'
    , description    varchar (4000)
    , CONSTRAINT pk_test_empty_table_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.test_empty_table'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 
<file_sep>#v1.2.6
#------------------------------------------------------------------------------
#  Purpose: 
#  export to_repl=<<the-string-to-replace-with>>
#------------------------------------------------------------------------------
doTestMorphDir(){
		
	do_log " INFO START : morph-dir.test"

	# doSpecMorphDir
	echo this function should not work without setting the following shell vars
	echo export dir_to_morph="<<the-dir-to-search-and-replace>>"
	echo to_srch="<<to_srch>>"
	echo to_repl="<<to_repl>>"

	sleep 1	
	bash src/bash/qto/qto.sh -a morph-dir
	echo now this test should succeed	
	mkdir -p /tmp/src/bash	
	cp -rv src/bash/qto /tmp/src/bash
	export dir_to_morph=/tmp/src/bash/qto
	export to_srch=qto
	export to_repl=new-app
	bash src/bash/qto/qto.sh -a morph-dir
	
	echo "and check the produced dir"
	find /tmp/src/bash/new-app -type f
	do_log " INFO STOP  : morph-dir.test"

}
#eof doMorphDir

#eof file: src/bash/qto/tests/dev/morph-dir.test.sh
<file_sep>#!/usr/bin/env python3
# usage: python src/python/generate-docx.py requirements_doc .
import sys
import os
import urllib.request, json 
import docx
from docx import Document
from docx.shared import Inches


table, path, app, db, user, ht_protocol, web_host, web_port = '', '', '', '', '', '', '', ''
def main():
    set_vars()
    tableData = getTableData()
    buildDoc(table,tableData)
    sys.exit()


def set_vars():
    try:
        global table, path, app, db, user, ht_protocol, web_host, web_port
        table = sys.argv[1]
        path = str(sys.argv[2] or '.')
        app = os.environ['postgres_app_db']
        db = os.environ['postgres_app_db']
        user= os.environ['postgres_app_usr']
        ht_protocol = os.environ['ht_protocol']
        web_host= os.environ['web_host']
        web_port = os.environ['port']
    except(IndexError) as error:
        print (error) 
        traceback.print_stack()
        sys.exit(1)

    return


def getTableData():
    try:
        url = ht_protocol + "://" + web_host + ":" + web_port + "/" + app + "/hiselect/" + table + "?pg-size=10000&bid=0"
        with urllib.request.urlopen(url) as url:
            data = json.loads(url.read().decode())
            print(data)
    except (Exception) as error:
        print(error)
    finally:
        return data['dat']


def buildDoc(table,tableData):
    doc = Document()
    #debug print('Loaded {}'.format(tableData))
    for row in tableData:
        print ('row {}'.format(row))
        level = 'Heading ' + str(row['level']+1)
        title = str(row['logical_order'] or '') + " " + row['name']
        if row['id'] == 0:
            doc.add_heading(title, 0)
            doc.add_section()
            doc.add_section()
        else:
            heading = doc.add_heading( title , row['level'])
            heading.style = doc.styles[level]
            p = doc.add_paragraph(row['description'])
            # https://stackoverflow.com/a/16859266/65706
            #if row['img_relative_path']:
            #    ip = document.add_paragraph()
            #    r = p.add_run()
            #    r.add_text(row['img_name'])
            #    r.add_picture('/tmp/foo.jpg')
            if row['src']:
                src = doc.add_paragraph(row['src'])
                src.style = doc.styles['Body Text 3']

    full_path=path + "/" + table.replace('_','.').replace('doc','docx')    
    doc.save(full_path)
    return 0


main()
<file_sep>-- DROP TABLE IF EXISTS category ; 

SELECT 'create the "category" table'
; 
   CREATE TABLE category (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , name           varchar (100) NOT NULL DEFAULT 'name...'
    , description    varchar (4000)
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_category_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

create unique index idx_uniq_category_id on category (id);

-- the rank search index
CREATE INDEX idx_rank_category ON category
USING gin(to_tsvector('English', name || ' ' || description || 'owner')); 


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.category'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 


--
-- TOC entry 3299 (class 0 OID 53411)
-- Dependencies: 268
-- Data for Name: category; Type: TABLE DATA; Schema: public; Owner: usrtstqtoadmin
--

COPY public.category (guid, id, name, description, update_time) FROM stdin;
cb989a12-d0b8-46e4-b2cc-5e2a974b5d29	200226071624	e2e	end-to-end 	2020-02-27 16:46:24
ed1bde66-694f-4e4d-816f-de594df31f48	200227185617	setup	the setup category	2020-02-27 16:46:33
9831b05b-5b54-4926-a08a-ac2b749d7bfb	200227185625	ui	The User Interface category	2020-02-27 16:46:47
9791074b-37f3-47b2-9af9-b2645a38e806	200227185640	provisioning	The provisioning category	2020-02-27 16:47:39
51c8b189-63f3-46c8-a666-f766b22563d9	200227185731	db 	The database related activities category	2020-02-27 16:47:57
70724562-d83c-446e-94cf-58ced84f3a0e	200227185858	unknown	The "unknown" category	2020-02-27 16:49:17
5b73486a-141f-491f-a2ee-919259151052	200227190115	security	The "security" category	2020-02-27 16:51:32
0d9722b9-3fc9-4d55-b6ff-1d9c73ebbbd9	200227190212	testing	Related heavily to testing	2020-02-27 16:52:39
aff0a804-f121-404d-b61c-3626eeeb57cc	200227190326	refactoring	All the tasks and activities are mainly refactoring related ...	2020-02-27 16:54:01
d049ae30-a13e-4a58-b530-47fe4bf10ea9	200227190501	functionality	A technical functionality	2020-02-27 16:55:22
641a8836-dceb-42f8-8eb3-c3e97539a650	200227190531	release	Release	2020-02-27 16:55:52
9e6a661c-c70c-4071-a14f-1a2e79b97f00	200227190652	docs	Related to documentation creation or update	2020-02-27 16:57:17
3b2b3a79-f778-4978-a2b0-03d02561be84	200227190843	feature	The feature category	2020-02-27 16:58:59
\.


--
-- TOC entry 3167 (class 2606 OID 53424)
-- Name: category category_id_key; Type: CONSTRAINT; Schema: public; Owner: usrtstqtoadmin
--

--The trigger:
CREATE TRIGGER trg_set_update_time_on_category BEFORE UPDATE ON category FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'category'::regclass;
<file_sep>#------------------------------------------------------------------------------
# usage example:
# source $PRODUCT_DIR/lib/bash/funcs/flush-screen.sh
# do_flush_screen
#------------------------------------------------------------------------------
do_flush_screen(){
   printf "\033[2J";printf "\033[0;0H"
}
<file_sep>-- call by: psql -d dev_qto -v PROCEDURE_NAME=func_fnc_get_all_app_user_roles < src/sql/pgsql/scripts/meta/show-procedure-definition.sql
SELECT      p.proname AS procedure_name,
--            p.pronargs AS num_args,
--            t1.typname AS return_type,
--            a.rolname AS procedure_owner,
--           l.lanname AS language_type,
--            p.proargtypes AS argument_types_oids,
            prosrc AS body
     FROM   pg_proc p
LEFT JOIN   pg_type t1 ON p.prorettype=t1.oid   
LEFT JOIN   pg_authid a ON p.proowner=a.oid 
LEFT JOIN   pg_language l ON p.prolang=l.oid
-- WHERE       proname = ':PROCEDURE_NAME' -- nope
--WHERE       proname like 'func%'
WHERE       proname = 'func_fnc_get_all_app_user_roles' --ok
;
<file_sep>#!/bin/bash
do_chk_provision_postgres(){

   do_export_json_section_vars $PRODUCT_DIR/cnf/env/$ENV.env.json '.env.db'
   # doScrambleConfs   # because well they were taken from public git repo

   psql_cnf_dir='/etc/postgresql/12/main' ; sudo mkdir -p "$psql_cnf_dir"

   set +e
   echo "$postgres_rdbms_usr:$postgres_rdbms_usr_pw" | chpasswd
   echo 'export PS1="`date "+%F %T"` \u@\h  \w \\n\\n  "' | sudo tee -a /var/lib/postgresql/.bashrc

   sudo /etc/init.d/postgresql restart

   sudo -Hiu postgres psql template1 -c 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp";'
   sudo -Hiu postgres psql template1 -c 'CREATE EXTENSION IF NOT EXISTS "pgcrypto";'
   sudo -Hiu postgres psql template1 -c 'CREATE EXTENSION IF NOT EXISTS "dblink";'

   test -f $psql_cnf_dir/pg_hba.conf && \
      sudo cp -v $psql_cnf_dir/pg_hba.conf $psql_cnf_dir/pg_hba.conf.orig.bak && \
      sudo cp -v $PRODUCT_DIR/cnf/postgres/$psql_cnf_dir/pg_hba.conf $psql_cnf_dir/pg_hba.conf && \
      sudo chown postgres:postgres $psql_cnf_dir

   test -f $psql_cnf_dir/postgresql.conf && \
      sudo cp -v $psql_cnf_dir/postgresql.conf $psql_cnf_dir/postgresql.conf.orig && \
      sudo cp -v $PRODUCT_DIR/cnf/postgres/$psql_cnf_dir/postgresql.conf $psql_cnf_dir/postgresql.conf

   sudo chown -R postgres:postgres "/etc/postgresql" && \
      sudo chown -R postgres:postgres "/var/lib/postgresql" && \
      sudo chown -R postgres:postgres "$psql_cnf_dir/pg_hba.conf" && \
      sudo chown -R postgres:postgres "$psql_cnf_dir/postgresql.conf"

   sudo /etc/init.d/postgresql restart

   expect <<- EOF_EXPECT
      set timeout -1
      spawn sudo -u $postgres_rdbms_usr psql --port $postgres_rdbms_port -c "\\\password"
      expect "Enter new password: "
      send -- "$postgres_rdbms_usr_pw\r"
      expect "Enter it again: "
      send -- "$postgres_rdbms_usr_pw\r"
      expect eof
EOF_EXPECT
}
<file_sep> ---- DROP TABLE IF EXISTS check_lists ; 

SELECT 'create the "check_lists" table'
; 
   CREATE TABLE check_lists (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , prio           integer NOT NULL DEFAULT 0
    , status         varchar (20) NOT NULL DEFAULT 'status?!...'
    , owner          varchar (30) NOT NULL DEFAULT 'kuka?!...'
    , name           varchar (200) NOT NULL DEFAULT 'mikä?..'
    , description    varchar (4000)
    , type           varchar (30) DEFAULT 'laji/tyyppi?...'
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_check_lists_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

create unique index idx_uniq_check_lists_id on check_lists (id);


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.check_lists'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 


SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'check_lists'::regclass;
<file_sep>#v1.0.7 
#------------------------------------------------------------------------------
# prints the usage of this script
#------------------------------------------------------------------------------
doPrintUsage(){

   source $PRODUCT_DIR/lib/bash/funcs/flush-screen.sh
   do_flush_screen

   while read -r f ; do
      acts="${acts:-} "$(echo `basename $f`| perl -ne 's/^(.*?)\.func.sh$/$1/g;print') ;
   done < <(ls -1 src/bash/$RUN_UNIT/funcs/*.func.sh)

   test_script=$(dirname $0)"/$RUN_UNIT.sh"
   do_flush_screen
   cat << EOF_USAGE | tee -a $log_file

------------------------------------------------------------------------
--   the main shell entry point of the $RUN_UNIT
--
------------------------------------------------------------------------
   Purpose:
	$RUN_UNIT - the main shell entry point for the $RUN_UNIT        

   Usage:
   $0 -a <<action-name-01>> -a <<ation-name-02>> -a <<action-name-03>>

   where <<action-name> is one of the following:
   $acts
	
	Qto CAN run against different qto projects :
   source lib/bash/funcs/export-json-section-vars.sh ; 
   do_export_json_section_vars <<some-dir>>/cnf/env/dev.env.json '.env.db'
   
		bash $0 -a to-tst
		bash $0 -a to-dev
		bash $0 -a to-prd
      bash $0 -a to-ver=1.0.5
      bash $0 -a to-app=<<new_app_name>>

------------------------------------------------------------------------

EOF_USAGE


   cat <<END_HELP


   #
   ## STOP  --- USAGE `basename $0`
   #------------------------------------------------------------------------------

END_HELP


}
#eof func doPrintUsage
<file_sep> ---- DROP TABLE IF EXISTS test_truncate_table ; 

SELECT 'create the "test_truncate_table" table'
;

   CREATE TABLE test_truncate_table (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , name           varchar (200) default 'name ...'
    , description    varchar (4000) default 'desc ...'
    , CONSTRAINT pk_test_truncate_table_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.test_truncate_table'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 


<file_sep># src/bash/qto/funcs/run-data-load-tests.help.sh

# v1.0.9
# ---------------------------------------------------------
# show the help for this shell action
# ---------------------------------------------------------
doHelpRunDataLoadTests(){

	do_log "DEBUG START doHelpRunDataLoadTests"
	
	cat doc/txt/qto/funcs/run-data-load-tests.func.txt
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!

	do_log "DEBUG STOP  doHelpRunDataLoadTests"
}
# eof func doHelpRunDataLoadTests


# eof file: src/bash/qto/funcs/run-data-load-tests.help.sh
<file_sep>-- credit & src: https://stackoverflow.com/a/42108936/65706
-- docs & usage @end
/*
SELECT grantee, privilege_type 
FROM information_schema.role_table_grants 
WHERE table_name='foobar'
;
*/
select a.tablename , b.usename
   , HAS_TABLE_PRIVILEGE(usename,tablename, 'select') as select
   , HAS_TABLE_PRIVILEGE(usename,tablename, 'insert') as insert
   , HAS_TABLE_PRIVILEGE(usename,tablename, 'update') as update
   , HAS_TABLE_PRIVILEGE(usename,tablename, 'delete') as delete
   , HAS_TABLE_PRIVILEGE(usename,tablename, 'references') as references  
FROM pg_tables a , pg_user b 
where a.tablename='monthly_issues'
;

/* 
usage: 
psql -d $postgres_app_db < src/sql/pgsql/scripts/check-postgres-tables-permissions.sql

*/

<file_sep># # usage: include it in your Makefile
# include <<file-path>>
#
# usage:
# make zip_me
# 


.ONESHELL: # Applies to every targets in the file!

.PHONY: zip_me ## @-> zip the whole project without the .git dir
zip_me:
	@clear
	-rm -v ../$(PRODUCT).zip ; zip -r ../$(PRODUCT).zip  . -x '*.git*'
	@sleep 1
	@clear
	@echo done check
	@echo $(PWD)/../$(PRODUCT).zip<file_sep>#!/bin/bash

encrypt_pass(){
 echo "$1"| perl -ne '$_=crypt($_,$_); s/[#\-%&\$*+()\/]//g; print substr (sprintf("%s%s", $_ , "12?"), 7, 9, "")'
}
c=10
while read -r user; do
	c=$((c+1))
	uid=$((1000+$c))
   echo runnnig sudo useradd -m -d "/home/$user" -s /bin/bash -c "$user" -u $uid -g 1000 $user
   set -x
   sudo useradd -m -d "/home/$user" -s /bin/bash -c "$user" -u $uid -g 1000 $user
   pass=$(encrypt_pass $user)
   sudo bash -c "echo $user:$pass | chpasswd"
	#echo and verify
	sudo cat /etc/passwd | grep --color=always "$user"
   echo "done creating :"
   echo "user: $user"
   echo "pass: $pass"
done < <(echo "$@"|tr ' ' "\n")
<file_sep>CREATE OR REPLACE FUNCTION "fnc_set_update_time" () RETURNS trigger AS '
	BEGIN
		NEW.update_time = DATE_TRUNC(''second'', NOW());
		RETURN NEW;
	END;'
LANGUAGE "plpgsql";

SELECT 'fnc_set_update_time exists is ' || exists(SELECT * FROM pg_proc WHERE proname='fnc_set_update_time')
;<file_sep>-- clear from data
TRUNCATE TABLE rep_userstories_doc;

-- copy selected columns from userstories_doc, only when status is set
INSERT INTO rep_userstories_doc (guid,id,status,prio,name,description,update_time)
SELECT guid,id,status,prio,name,description,update_time
FROM userstories_doc
WHERE 1=1 AND status!='status...';

-- make 03-wip standard instead of 03-act and 03-active
UPDATE rep_userstories_doc
SET status='03-wip'
WHERE status ILIKE '03-a%';<file_sep># src/bash/qto/funcs/remove-package-files.help.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doHelpRemovePackageFiles comments ...
# ---------------------------------------------------------
doHelpRemovePackageFiles(){

	do_log "DEBUG START doHelpRemovePackageFiles"
	
	cat doc/txt/qto/helps/remove-package-files.help.txt
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!

	do_log "DEBUG STOP  doHelpRemovePackageFiles"
}
# eof func doHelpRemovePackageFiles


# eof file: src/bash/qto/funcs/remove-package-files.help.sh
<file_sep>-- DROP TABLE IF EXISTS questions_status ; 

SELECT 'create the "questions_status" table'
; 
   CREATE TABLE questions_status (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , name           varchar (100) NOT NULL DEFAULT 'name...'
    , description    varchar (4000)
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_questions_status_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

create unique index idx_uniq_questions_status_id on questions_status (id);

-- the rank search index
CREATE INDEX idx_rank_questions_status ON questions_status
USING gin(to_tsvector('English', name || ' ' || description || 'owner')); 


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.questions_status'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 


-- Data for Name: questions_status; Type: TABLE DATA; Schema: public; Owner: usrdevqtoadmin
--

COPY public.questions_status (guid, id, name, description, update_time) FROM stdin;
cb989a14-d0b8-46e4-b2cc-5e2a974b5d29	200226071624	01-eval	The question is being evaluated	2020-02-26 07:16:45
94ae9eac-e6df-4193-9d36-a527d5897292	200226092430	02-todo	The questions has to be done	2020-02-26 07:17:02
d0b9476b-7eb0-4045-b172-91c8635859d0	200226092448	03-wip	The question is in Work in Progress	2020-02-26 07:17:16
652ccaad-89a4-4e5e-bb1b-6859ab3472f4	200226092506	04-diss	The must be discarded	2020-02-26 07:17:36
a9224b60-506a-457c-8303-27345b5a5a09	200226092521	05-tst	The question is being tested	2020-02-26 07:17:48
9594f01f-92ef-48da-9319-57792ac8142f	200226092532	06-onhold	The question has been put on hold	2020-02-26 07:17:59
a45ee85b-570c-4661-b8f4-cd39f5d5ff76	200226092543	07-qas	The questions  is being tested for quality	2020-02-26 07:18:16
91ecf204-43d5-4ac5-aa5f-f1a0eee1ad53	200226092559	08-post	The question is being postponed for completion in later point of time 	2020-02-26 07:18:56
e486d2d7-0789-4af2-8466-9b8c03743d85	200226092640	09-done	The question has been done	2020-02-26 07:19:08
c36ac35d-d010-41ce-84f0-31ba5d808aa8	200226093618	03-act	The questions is being actively worked on 	2020-02-26 07:30:54
\.

--The trigger:
CREATE TRIGGER trg_set_update_time_on_questions_status BEFORE UPDATE ON questions_status FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'questions_status'::regclass;
<file_sep># src/bash/qto/funcs/remove-package.help.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doHelpRemovePackage comments ...
# ---------------------------------------------------------
doHelpRemovePackage(){

	do_log "DEBUG START doHelpRemovePackage"
	
	cat doc/txt/qto/helps/remove-package.help.txt
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!

	do_log "DEBUG STOP  doHelpRemovePackage"
}
# eof func doHelpRemovePackage


# eof file: src/bash/qto/funcs/remove-package.help.sh
<file_sep># src/bash/qto/funcs/run-functional-tests.func.sh

# ---------------------------------------------------------
# implement the calls to all the functional tests
# ---------------------------------------------------------
doRunFunctionalTests(){

   export QTO_NO_AUTH=1
   test -z "${PROJ_INSTANCE_DIR-}" && PROJ_INSTANCE_DIR="$PRODUCT_DIR"
   #
   do_export_json_section_vars $PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json '.env.db'

   do_log "INFO START testing controllers"
   while read -r f ; do
      do_log "INFO START functional test for $f"
      perl $f ;
      test $? -ne 0 && do_exit $? " the tests in the $f failed !!!"
      do_log "INFO STOP  functional test for $f"
      sleep 1
      do_flush_screen
   done < <(find src/perl/qto/t/lib/Qto/Controller -type f -name '*.t' |sort)

   export QTO_NO_AUTH=0
}
<file_sep>#!/usr/bin/env bash
set -x

echo from install-test PRODUCT_DIR: $PRODUCT_DIR
echo from install-test hostname $host_host_name
<file_sep>find_me(){
   to_srch=$1
   find src -type f -exec grep -nHi --color=always "$to_srch" {} \; | less -R
}
<file_sep>-- DROP TABLE IF EXISTS problems_status ; 

SELECT 'create the "problems_status" table'
; 
   CREATE TABLE problems_status (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , name           varchar (100) NOT NULL DEFAULT 'name...'
    , description    varchar (4000)
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_problems_status_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

create unique index idx_uniq_problems_status_id on problems_status (id);

-- the rank search index
CREATE INDEX idx_rank_problems_status ON problems_status
USING gin(to_tsvector('English', name || ' ' || description || 'owner')); 


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.problems_status'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 


-- Data for Name: problems_status; Type: TABLE DATA; Schema: public; Owner: usrdevqtoadmin
--

COPY public.problems_status (guid, id, name, description, update_time) FROM stdin;
cb989a14-d0b8-46e4-b2cc-5e2a974b5d29	200226071624	01-eval	The problem is being evaluated	2020-02-26 07:16:45
94ae9eac-e6df-4193-9d36-a527d5897292	200226092430	02-todo	The problems has to be done	2020-02-26 07:17:02
d0b9476b-7eb0-4045-b172-91c8635859d0	200226092448	03-wip	The problem is in Work in Progress	2020-02-26 07:17:16
652ccaad-89a4-4e5e-bb1b-6859ab3472f4	200226092506	04-diss	The must be discarded	2020-02-26 07:17:36
a9224b60-506a-457c-8303-27345b5a5a09	200226092521	05-tst	The problem is being tested	2020-02-26 07:17:48
9594f01f-92ef-48da-9319-57792ac8142f	200226092532	06-onhold	The problem has been put on hold	2020-02-26 07:17:59
a45ee85b-570c-4661-b8f4-cd39f5d5ff76	200226092543	07-qas	The problems  is being tested for quality	2020-02-26 07:18:16
91ecf204-43d5-4ac5-aa5f-f1a0eee1ad53	200226092559	08-post	The problem is being postponed for completion in later point of time 	2020-02-26 07:18:56
e486d2d7-0789-4af2-8466-9b8c03743d85	200226092640	09-done	The problem has been done	2020-02-26 07:19:08
c36ac35d-d010-41ce-84f0-31ba5d808aa8	200226093618	03-act	The problems is being actively worked on 	2020-02-26 07:30:54
\.

--The trigger:
CREATE TRIGGER trg_set_update_time_on_problems_status BEFORE UPDATE ON problems_status FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'problems_status'::regclass;
<file_sep># src/bash/qto/funcs/clone-project.func.sh

# v1.0.9
# ---------------------------------------------------------
# cat doc/txt/qto/funcs/clone-project.func.txt
# ---------------------------------------------------------
doCloneProject(){

	do_log "DEBUG START doCloneProject"
	
	sleep "$sleep_interval"
   # doc/txt/qto/spec/clone-project.spec.txt
   export src_proj='ysg-issues'
   export tgt_proj='phz'
   export global_cnf_dir='/vagrant/csitea/cnf/projects/qto'

   # load the src project configuration file
   doParseIniEnvVars "$global_cnf_dir"'/'"$src_proj"".prd.host-name.cnf"

   # go to the ysg projects daily data root dir
   cd $mix_data_dir

   # define yesterday 
   export tgt_date=$(date --date="-1 day" "+%Y-%m-%d")

   # create a zip package of the yester-day day 
   while read -r f ; do zip $tgt_date.zip $f ; \
   done < <(find "$(date "+%Y" -d "$tgt_date")"'/'$(date "+%Y-%m" -d "$tgt_date")/$(date "+%Y-%m-%d" -d "$tgt_date"))

   # load the newly defined cloned project dir
   doParseIniEnvVars "$global_cnf_dir"'/'"$tgt_proj"".prd.host-name.cnf"

   # create the new projects daily data root dir
   mkdir -p $mix_data_dir
   export dir_to_morph=$mix_data_dir

   # unzip the yesterday daily dir for the new project
   unzip -o $tgt_date.zip -d $mix_data_dir/

   # go back to the product instance dir
   cd -

   # morph the dir
   export to_srch=$src_proj
   export to_repl=$tgt_proj
   bash src/bash/qto/qto.sh -a morph-dir

   # start working again on the project to be cloned
   doParseIniEnvVars "$global_cnf_dir"'/'"$tgt_proj"".prd.host-name.cnf"

   # and increase the dat
   bash src/bash/qto/qto.sh -a increase-date -d today

	do_log "DEBUG STOP  doCloneProject"
}
# eof func doCloneProject


# eof file: src/bash/qto/funcs/clone-project.func.sh
<file_sep>#!/bin/bash

do_fix_db_permissions(){

   do_setup_db_roles

   test -z "${PROJ_INSTANCE_DIR-}" && PROJ_INSTANCE_DIR="$PRODUCT_DIR"

   test -z $PROJ_CONF_FILE && PROJ_CONF_FILE=$PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json
   do_export_json_section_vars $PROJ_CONF_FILE '.env.db'

   do_log "INFO using PROJ_INSTANCE_DIR: $PROJ_INSTANCE_DIR" ;
   do_log "INFO using PROJ_CONF_FILE: $PROJ_CONF_FILE"

   cd /tmp
   sudo -u postgres PGPASSWORD=$postgres_usr_pw psql --port $postgres_rdbms_port --host $postgres_rdbms_host -c "
      grant all privileges on database $postgres_app_db to $postgres_sys_usr_admin" ;

   sudo -u postgres PGPASSWORD=$postgres_usr_pw psql --port $postgres_rdbms_port \
      --host $postgres_rdbms_host -d $postgres_app_db -c "
      ALTER USER $postgres_app_usr WITH PASSWORD '"$<PASSWORD>"'";

   PGPASSWORD=$postgres_sys_usr_admin_pw psql -U "${postgres_sys_usr_admin:-}" \
      --host $postgres_rdbms_host --port $postgres_rdbms_port -d "$postgres_app_db" \
      -c "  GRANT SELECT,INSERT,UPDATE,DELETE,TRUNCATE ON ALL TABLES IN SCHEMA public TO $postgres_app_usr;
            GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO $postgres_app_usr"

   sudo -u postgres PGPASSWORD=$postgres_usr_pw psql \
      --port $postgres_rdbms_port --host $postgres_rdbms_host -d "$postgres_app_db" -c \
         "GRANT SELECT,INSERT,UPDATE,DELETE,TRUNCATE ON ALL TABLES
         IN SCHEMA public TO $postgres_app_usr;
         GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO $postgres_app_usr"
   sudo -u postgres PGPASSWORD=$postgres_usr_pw psql \
      --port $postgres_rdbms_port --host $postgres_rdbms_host -d "$postgres_app_db" -c \
         "GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO $postgres_app_usr"
   cd -

}
<file_sep>#!/usr/bin/env bash 
# source & courtesy of:
# https://github.com/mislav/dotfiles/blob/master/bin/tmux-session
# TODO: persist and restore_tmux_session the state & position of panes.


doTestRestoreTmuxSession(){

	doCheckTmuxIsInstalled

	tmux start-server
	local count=0
	local dimensions="$(terminal_size)"

	while IFS=$'\t' read session_name window_name dir; do
	 if [[ -d "$dir" && $window_name != "log" && $window_name != "man" ]]; then
		if session_exists "$session_name"; then
		  add_window "$session_name" "$window_name" "$dir"
		else
		  new_tmux_session "$session_name" "$window_name" "$dir" "$dimensions"
		  count=$(( count + 1 ))
		fi
	 fi
	done < ~/.tmux-session
	
	do_log "restore_tmux_sessiond $count sessions"
	tmux list-sessions | column -t | sort

}
#eof test doRestoreTmuxSession
<file_sep>-- how-to move issues from one time frame to another 
INSERT INTO monthly_issues (guid,id,prio,status,category,name,description,type,owner,update_time )
SELECT guid,id,prio,status,category,name,description,type,owner,update_time 
FROM release_issues  
WHERE 1=1 AND status='09-done'
ON CONFLICT (id) DO UPDATE
SET guid = excluded.guid ,id = excluded.id ,prio = excluded.prio ,status = excluded.status ,category = excluded.category ,name = excluded.name ,description = excluded.description ,type = excluded.type ,owner = excluded.owner ,update_time = excluded.update_time
;
INSERT INTO yearly_issues (guid,id,prio,status,category,name,description,type,owner,update_time )
SELECT guid,id,prio,status,category,name,description,type,owner,update_time 
FROM monthly_issues  
WHERE 1=1 AND status='09-done'
ON CONFLICT (id) DO UPDATE
SET guid = excluded.guid ,id = excluded.id ,prio = excluded.prio ,status = excluded.status ,category = excluded.category ,name = excluded.name ,description = excluded.description ,type = excluded.type ,owner = excluded.owner ,update_time = excluded.update_time
;

-- and then delete them from the source table 
DELETE FROM release_issues 
WHERE 1=1
AND status='09-done'
; 


SELECT column_name
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name   = 'monthly_issues'
;
INSERT INTO daily_issues SELECT * FROM monthly_issues  WHERE status = '02-todo-copy-to-daily' 
;
UPDATE daily_issues set status = '02-todo' where status = '02-todo-copy-to-daily' 
;
<file_sep>place the src of your qto / apps below this dir. Note the run-time subdirs ...
<file_sep>-- DROP TABLE IF EXISTS contacts ; 

SELECT 'create the "contacts" table'
; 
   CREATE TABLE contacts (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , name           varchar (100) NOT NULL DEFAULT 'name...'
    , email          varchar (60) NOT NULL DEFAULT 'email'
    , mobile         varchar (30) NOT NULL DEFAULT 'mobile'
    , description    varchar (4000)
    , organisation   varchar (50) NOT NULL DEFAULT 'organisation'
    , first_name     varchar (50) NOT NULL DEFAULT 'first_name...'
    , last_name      varchar (50) NOT NULL DEFAULT 'last_name...'
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_contacts_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

create unique index idx_uniq_contacts_id on contacts (id);

-- the rank search index
CREATE INDEX idx_rank_contacts ON contacts
USING gin(to_tsvector('English', name || ' ' || description || 'owner')); 


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.contacts'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 



--The trigger:
CREATE TRIGGER trg_set_update_time_on_contacts BEFORE UPDATE ON contacts FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'contacts'::regclass;
<file_sep>FROM alpine
# based on alpine-perl of: https://github.com/krumeich

ARG ENV
ARG UID
ARG GID
ARG PRODUCT
ARG ORG

ENV PRODUCT=$PRODUCT
ENV ORG=$ORG
ENV APPUSR=appusr
ENV APPGRP=appgrp
ENV PS1='`date "+%F %T"` \u@\h  \w \n\n  '
ENV PRODUCT_DIR="/opt/$PRODUCT"
ENV EDITOR="vim"
ENV ENV=$ENV

VOLUME $PRODUCT_DIR


# install OS utils
RUN apk update && apk upgrade && apk add --no-cache \
    bash binutils vim jq wget curl zip unzip tar busybox-extras su-exec sudo shadow


# install build utils && backend-utils
# gnupg is needed for Module::Signature.
RUN apk update && apk upgrade && apk add --no-cache \
    build-base make gcc openssl-dev libmagic ttf-freefont gnupg libpq libpq-dev


# start ::: adding OS APPUSR and APPGRP
RUN test -z $(getent group $GID | cut -d: -f1) || \
        groupmod -g $((GID+1000)) $(getent group $GID | cut -d: -f1)

# create a APPGRP and APPUSR
RUN set -x ; addgroup -g "$GID" -S "$APPGRP" && \
  adduser \
  --disabled-password \
  -g "$GID" \
  -D \
  -s "/bin/bash" \
  -h "/home/$APPUSR" \
  -u "$UID" \
  -G "$APPGRP" "$APPUSR" && exit 0 ; exit 1

RUN echo "$APPUSR ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers


# stop ::: adding OS APPUSR and APPGRP


RUN mkdir -p /usr/src/perl

WORKDIR /usr/src/perl

## from perl; `true make test_harness` because 3 tests fail
## some flags from http://git.alpinelinux.org/cgit/aports/tree/main/perl/APKBUILD?id=19b23f225d6e4f25330e13144c7bf6c01e624656
RUN curl -SLO https://www.cpan.org/src/5.0/perl-5.32.0.tar.gz \
    && echo 'ddecb3117c016418b19ed3a8827e4b521b47d6bb *perl-5.32.0.tar.gz' | sha1sum -c - \
    && tar --strip-components=1 -xzf perl-5.32.0.tar.gz -C /usr/src/perl \
    && rm perl-5.32.0.tar.gz \
    && ./Configure -des \
        -Duse64bitall \
        -Dcccdlflags='-fPIC' \
        -Dcccdlflags='-fPIC' \
        -Dccdlflags='-rdynamic' \
        -Dlocincpth=' ' \
        -Duselargefiles \
        -Dusethreads \
        -Duseshrplib \
        -Dd_semctl_semun \
        -Dusenm \
    && make libperl.so \
    && make -j$(nproc) \
    && TEST_JOBS=$(nproc) make test_harness \
    && make install \
    && curl -LO https://raw.githubusercontent.com/miyagawa/cpanminus/master/cpanm \
    && chmod +x cpanm \
    && ./cpanm App::cpanminus \
    && rm -fr ./cpanm /root/.cpanm /usr/src/perl

## from tianon/perl
ENV PERL_CPANM_OPT --verbose --mirror https://cpan.metacpan.org --mirror-only
RUN cpanm Digest::SHA Module::Signature && \
  cpanm local::lib && \
rm -rf ~/.cpanm

ENV PERL_CPANM_OPT $PERL_CPANM_OPT --verify

RUN echo "export PS1=\"$PS1\"" >> /home/$APPUSR/.bashrc && \
  chown -R $APPUSR:$APPGRP /home/$APPUSR && \
  chmod -R 0775 /home/$APPUSR

ADD --chown=$APPUSR:$APPGRP "." /home/$APPUSR$PRODUCT_DIR
# switch to the app user
USER $APPUSR


# stop ::: adding OS APPUSR and APPGRP

WORKDIR $PRODUCT_DIR

# Note: using $APPUSR/$PRODUCT_DIR will result in /home/appusr//opt/product/r
#                                   ~~~^^^~~~
RUN bash /home/$APPUSR$PRODUCT_DIR/run -a do_alpine_chk_install_perl_modules



CMD exec /bin/bash -c /home/$APPUSR$PRODUCT_DIR/src/bash/run/docker-init-app.sh

<file_sep>#!/usr/bin/env bash

# start install perl modules
wget - http://cpanmin.us | perl - --self-upgrade

# src: http://stackoverflow.com/a/13090364/65706src: 
# echo y|cpan
# echo o conf prerequisites_policy follow | cpan
# echo o conf commit | cpan

#bash -c " -s 'curl -L cpanmin.us | perl - Mojolicious'"
# cpan Mojolicious
# stop install perl modules

<file_sep># src/bash/qto/funcs/remove-package-files.func.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doRemovePackageFiles comments ...
# ---------------------------------------------------------
doRemovePackageFiles(){

	do_log "DEBUG START doRemovePackageFiles"
	
	cat doc/txt/qto/funcs/remove-package-files.func.txt
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!

	do_log "DEBUG STOP  doRemovePackageFiles"
}
# eof func doRemovePackageFiles


# eof file: src/bash/qto/funcs/remove-package-files.func.sh
<file_sep># ---------------------------------------------------------
# create a new <<env-type>> from this product instance
# bash src/bash/qto/qto.sh -a to-dev
# bash src/bash/qto/qto.sh -a to-tst
# bash src/bash/qto/qto.sh -a to-prd
# ---------------------------------------------------------
doChangeEnvType(){
   tgt_env="${1:-}"
   prefix='to-env='
   tgt_env=${tgt_env#$prefix}


	tgt_environment_name=$(echo $environment_name | perl -ne "s/$ENV/$tgt_env/g;print")
	tgt_PRODUCT_DIR=$product_dir/$tgt_environment_name

	test "$tgt_env" == "$ENV" && return
	# remove everything from the tgt product version dir - no extra files allowed !!!
	mkdir -p $tgt_PRODUCT_DIR && test -d $tgt_PRODUCT_DIR && \
      mv -v $tgt_PRODUCT_DIR $tgt_PRODUCT_DIR.$(date "+%Y%m%d_%H%M%S")
	test $? -eq 0  || do_exit 1 "cannot write to $tgt_PRODUCT_DIR !"
	
   mkdir -p $tgt_PRODUCT_DIR	
	test $? -ne 0 && do_exit 1 "Failed to create $tgt_PRODUCT_DIR !"
	
	doCreateRelativePackage
	unzip -o $zip_file -d $tgt_PRODUCT_DIR
	cp -v $zip_file $tgt_PRODUCT_DIR


   perl -pi -e 's|'$ENV'|'$tgt_env'|g' $tgt_PRODUCT_DIR/.env

   cd $tgt_PRODUCT_DIR
   ln -sf `pwd`/src/bash/qto/qto.sh qto
   ln -sf `pwd`/src/perl/qto/script/qto.pl qto.pl
   cd -
}
<file_sep># # usage: include it in your Makefile
# include lib/make/build-img-func.mk
#
# # function usage example:
#
# install-web-node:
# 	$(call install-img,web-node,80)

include lib/make/demand-var.func.mk
include lib/make/stop-img-containers.func.mk

define install-img
	@clear
	$(call demand-var,ORG)
	$(call demand-var,APP)
	$(call demand-var,ENV)

	@PORT_COMMAND=`[[ "${2}" == "" ]] || echo "--publish ${2}:${2}"`
	@NO_CACHE=`[[ "${3}" == "" ]] || echo "--no-cache"`

	docker build . -t ${product}-$(1)-img $${NO_CACHE}\
		--build-arg UID=$(shell id -u) \
		--build-arg GID=$(shell id -g) \
		--build-arg ORG=$(ORG) \
		--build-arg APP=$(APP) \
		--build-arg ENV=$(ENV) \
		--build-arg PRODUCT=${PRODUCT} \
		-f src/docker/$(1)/Dockerfile

	$(call uninstall-img,$1)

	docker run -it -d --restart=always $${PORT_COMMAND}\
		-v $$(pwd):/opt/${PRODUCT} \
		-v $$HOME/.aws:/home/appusr/.aws \
		-v $$HOME/.ssh:/home/appgrp/.ssh $${PORT_COMMAND}\
		--name ${product}-$(1)-con ${product}-${1}-img ;
	@echo -e to attach run: "\ndocker exec -it ${product}-${1}-con /bin/bash"
	@echo -e to get help run: "\ndocker exec -it ${product}-${1}-con ./run --help"

endef
<file_sep># src/bash/qto/funcs/run-data-load-tests.test.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doTestRunDataLoadTests comments ...
# ---------------------------------------------------------
doTestRunDataLoadTests(){
	do_log "DEBUG START doTestRunDataLoadTests"
	
	
	sleep "$sleep_interval"
	# Action !!!
   clear ; bash src/bash/qto/qto.sh -a run-data-load-tests

	do_log "DEBUG STOP  doTestRunDataLoadTests"
}
# eof func doTestRunDataLoadTests


# eof file: src/bash/qto/funcs/run-data-load-tests.test.sh
<file_sep>#!/usr/bin/env bash
# file: src/bash/qto/install-prerequisites-on-ubuntu.sh

# caveat package names are for Ubuntu !!!
set -eu -o pipefail # fail on error , debug all lines

# run as root
[ "$USER" = "root" ] || exec sudo "$0" "$@"

echo "=== $BASH_SOURCE on $(hostname -f) at $(date)" >&2


echo installing the must-have pre-requisites
while read -r p ; do
	if [ "" == "`which $p`" ];
	then echo "$p Not Found";
		if [ -n "`which apt-get`" ];
		then apt-get install -y $p ;
		elif [ -n "`which yum`" ];
		then yum -y install $p ;
		fi ;
	fi
done < <(cat << "EOF"
	perl
	zip
	unzip
	exuberant-ctags
	mutt
	curl
	wget
	libwww-curl-perl
   libtest-www-selenium-perl
	postgresql-11.3
	libdbd-pgsql
	libxml-atom-perl
   git
   vim
   libxml-atom-perl
   tar
   gzip
   graphviz
   python-selenium chromium-chromedriver
   python-selenium
	python-setuptools 
	python-dev 
	build-essential
   gpgsm
   nodejs
   lsof
   pgrep
   minio
   mc
   libssl-dev
EOF
)

# this one is for the gpgsm binary
sudo apt-mark auto gpgsm

# to install selenium
sudo easy_install pip
sudo pip install --upgrade selenium

curl -L http://cpanmin.us | perl - --sudo App::cpanminus
sudo cpanm install JSON

# js libraries
sudo npm install -g mocha

echo installing the nice-to-have pre-requisites
echo you have 5 seconds to proceed ...
echo or
echo hit Ctrl+C to quit
echo -e "\n"
sleep 6

echo installing the nice to-have pre-requisites
while read -r p ; do
	if [ "" == "`which $p`" ];
	then echo "$p Not Found";
		if [ -n "`which apt-get`" ];
		then apt-get install -y $p ;
		elif [ -n "`which yum`" ];
		then yum -y install $p ;
		fi ;
	fi
done < <(cat << "EOF"
	tig
EOF
)

echo installing Mojolicious
sudo curl -L cpanmin.us | perl - Mojolicious

# cd src/perl/qto/public

# eof file: src/bash/qto/install-prerequisites-on-ubuntu.sh
<file_sep>
TRUNCATE TABLE app_items_roles_permissions ;
INSERT INTO app_items_roles_permissions ( app_roles_guid,app_items_guid,app_routes_guid,name,description) 
   SELECT app_roles.guid , t2.guid, t3.guid
		, app_roles.name || '__may__' || t3.name || '__' || t2.name as name
		, 'WHETHER OR NOT THE ' || app_roles.name || ' CAN ' || t3.name || ' THE ' || t2.name as name
   from app_roles
   cross join (
      SELECT app_items.guid , app_items.name 
         FROM app_items
         WHERE 1=1 
         and app_items.name not like 'test_%'
         and app_items.name not like '%_2018%'
         and app_items.name not like '%_2018'
         and app_items.name not like '%_2019%'
         and app_items.name not like '%_2019'
         and app_items.name not like '%_2020%'
         and app_items.name not like '%_2020'
         ) t2
   cross join (
      SELECT app_routes.guid , app_routes.name FROM app_routes) t3
;

-- the query and search routes apply to all the items - should use the all fake item
DELETE FROM app_items_roles_permissions WHERE 1=1 
   AND app_items_guid NOT IN 
   ( SELECT guid from app_items WHERE name = 'all')
   AND app_routes_guid IN 
   ( SELECT guid from app_routes WHERE name IN ('query' , 'search'))
; 

-- deny anything related to app_roles and users to all but the product instance owner
UPDATE app_items_roles_permissions 
   set allowed=false
   , name = replace(name, 'may', 'mayNOT')
WHERE 1=1
AND ( name not like '%userstories%')
AND ( name like '%app_roles%' or name like '%users%' or name like '%app_items_roles_permissions%' 
      or name like '%app_items%' or name like '%app_routes%' 
    )
;

-- the PRODUCT_INSTANCE_OWNER MUST see EVERYTHING
UPDATE app_items_roles_permissions set 
   allowed=true
   , name = replace(name, 'mayNOT', 'may')
WHERE 1=1
   AND app_roles_guid IN 
   ( SELECT guid from app_roles WHERE name = 'PRODUCT_INSTANCE_OWNER')
;

-- the select-col must be available for everyone logged in
UPDATE app_items_roles_permissions set 
   allowed=true
   , name = replace(name, 'mayNOT', 'may')
WHERE 1=1
   AND app_routes_guid IN 
   ( SELECT guid from app_routes WHERE name = 'select-col')
;

-- any combination of view with item not ending with _doc is pointless 
DELETE FROM app_items_roles_permissions 
WHERE 1=1
AND ( name like '%::view%' and name not like '%_doc')
; 

-- and display the "forbidden ones"
SELECT app_roles.name , app_items_roles_permissions.allowed , app_routes.name , app_items.name 
FROM app_items_roles_permissions
LEFT JOIN app_roles ON app_roles.guid = app_roles_guid
LEFT JOIN app_routes ON app_routes.guid = app_routes_guid
LEFT JOIN app_items ON app_items.guid = app_items_guid
WHERE app_items_roles_permissions.allowed = 'false'
;

-- all open routes must be accesible
UPDATE app_items_roles_permissions set 
   allowed=true
   , name = replace(name, 'mayNOT', 'may')
WHERE 1=1
   AND app_routes_guid IN 
   ( SELECT guid from app_routes WHERE app_routes.is_open = true)
;

-- the select-col route should be available to all the roles
UPDATE app_items_roles_permissions set 
   allowed=true
   , name = replace(name, 'mayNOT', 'may')
WHERE 1=1
   AND app_routes_guid IN 
   ( SELECT guid from app_routes WHERE app_routes.name = 'select-col')
;

UPDATE app_routes set is_open_in = True where name in ('query', 'search');
UPDATE app_routes set is_open = True where name in ('logon', 'login');

<file_sep>doRunPgsqlScripts(){
   test -z "${PROJ_INSTANCE_DIR-}" && export PROJ_INSTANCE_DIR="$PRODUCT_DIR"

   test -z ${PROJ_CONF_FILE:-} && export PROJ_CONF_FILE="$PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json"

   pgsql_scripts_dir="$PRODUCT_DIR/src/sql/pgsql/qto"
   tmp_log_file="$tmp_dir/.$$.log"
   do_export_json_section_vars $PROJ_CONF_FILE '.env.db'
   do_log "INFO using PROJ_INSTANCE_DIR: $PROJ_INSTANCE_DIR"
   do_log "INFO using PROJ_CONF_FILE: $PROJ_CONF_FILE"
   sleep 3 ; do_flush_screen ;
   echo -e "should run the following files: \n\n" ; find "$pgsql_scripts_dir" -type f -name "*.sql"|sort -n

   # run the sql scripts in alphabetical order - hence the file naming convention
   while read -r sql_script ; do

      relative_sql_script=$(echo $sql_script|perl -ne "s|$PROJ_INSTANCE_DIR\/||g;print")
      do_log "INFO START ::: running $relative_sql_script" ; echo -e '\n\n'
      perl -pi -e 's|-- DROP|DROP|g' $sql_script # drop and create the objects

      set -x
      PGPASSWORD="${postgres_sys_usr_admin_pw:-}" psql -v ON_ERROR_STOP=1 -q -t -X -w \
         -h $postgres_rdbms_host -p $postgres_rdbms_port -U "${postgres_sys_usr_admin:-}" \
         -v AdminEmail="${AdminEmail:-}" \
         -v postgres_app_db="$postgres_app_db" -f "$sql_script" "$postgres_app_db" > "$tmp_log_file" 2>&1
      ret=$?
      set +x

      perl -pi -e 's|DROP|-- DROP|g' $relative_sql_script
      cat "$tmp_log_file" ; cat "$tmp_log_file" >> $log_file  # show it and save it
      test $ret -ne 0 && sleep 3
      test $ret -ne 0 && do_exit 1 "pid: $$ psql ret $ret - failed to run sql_script: $sql_script !!!"
      test $ret -ne 0 && break

      echo -e '\n\n'; do_flush_screen ;
      do_log "INFO STOP   ::: running $relative_sql_script"
   done < <(find "$pgsql_scripts_dir" -type f -name "*.sql"|sort -n)

   do_flush_screen ;
}
<file_sep>doPrintStats(){
   echo -e "\nprint the line numbers per file type \n"
   echo cnf md txt html css sql html.ep js.html.ep sh pm pl t | tr ' ' "\n" | \
   while read ftype ; do lines=$(find . -name "*.$ftype" -exec cat {} + | wc -l); \
   printf "%-20s %12d \n" $ftype $lines ; done | sort -nr -k 2

}
<file_sep>#!/bin/bash
# ---------------------------------------------------------
# cat cnf/qto.dev.host-name.cnf
# [MainSection]
# postgres_db_name     = dev_qto
# postgres_db_host     = host-name
#
# call by:
# source lib/bash/funcs/parse-ini-section-vars.func.sh ; do_parse_ini_section_vars $AWS_SHARED_CREDENTIALS_FILE "profile $AWS_PROFILE"
# ---------------------------------------------------------
do_parse_ini_section_vars(){

   cnf_file=$1;shift 1;
   INI_SECTION=$1;shift 1;

   test -z "$cnf_file" && echo " you should set the cnf_file !!!"

   ( set -o posix ; set ) | sort >~/vars.before

   eval `sed -e 's/[[:space:]]*\=[[:space:]]*/=/g' \
      -e 's/#.*$//' \
      -e 's/[[:space:]]*$//' \
      -e 's/^[[:space:]]*//' \
      -e "s/^\(.*\)=\([^\"']*\)$/export \1=\"\2\"/" \
      < $cnf_file \
      | sed -n -e "/^\[$INI_SECTION\]/,/^\s*\[/{/^[^#].*\=.*/p;}"`

   # and post-register for nice logging
   ( set -o posix ; set ) | sort >~/vars.after


   echo "INFO added the following vars from section: [$INI_SECTION]"
   comm -3 ~/vars.before ~/vars.after | perl -ne 's#\s+##g;print "\n $_ "'
}
<file_sep># src/bash/qto/funcs/generate-sql.func.sh

# v1.2.0
# ---------------------------------------------------------
# read the lst file for each var in headers to copy template
# into a new %script%.sql search and replaces vars with 
# values from the lst file both in file names and in content
# ---------------------------------------------------------
doGenerateSQL(){

	do_log "DEBUG START doGenerateSQL"
	
   while read -r list_file ; do   #foreach list file in the dat/lst/sql dir

      cd $PRODUCT_DIR/src/sql/mysql/run
         
      # foreach sql template
      while read -r sql_template ; do

         headers=$(head -n 1 "$list_file")
         TMP_IFS=$IFS
         
         while read -r line ; do

            sql_file=$sql_template

            IFS=$' \t\n' read -r -a column_names <<< "$headers"
            #debug ok printf '%s\n' "${column_names[@]}"

            IFS=$' \t\n' read -r -a tokens <<< "$line"
            #debug ok printf '%s\n' "${tokens[@]}"

            for index in "${!column_names[@]}"
            do
                to_srch="${column_names[$index]}"
                to_repl="${tokens[$index]}"
                sql_file=$(echo "$sql_file"|perl -ne 's/(.*)\.SQL/$1.sql/g;print')
                sql_file=$(echo "$sql_file"|perl -ne 's{%'"$to_srch"'%}{'"$to_repl"'}g;print')
            done
            cp -v "$sql_template" "$sql_file"
            
            for index in "${!column_names[@]}"
            do
                to_srch="${column_names[$index]}"
                to_repl="${tokens[$index]}"
                perl -pi -e "use utf8 ; binmode STDOUT, \":utf8\"; 
                use open':encoding(utf8)';s{%""$to_srch""%}{""$to_repl""}g;" "$sql_file"
            done
          
         done < <(tail -n +2 "$list_file") # cat all but the first line 
      done < <(find $PRODUCT_DIR/src/sql/mysql/run -type f -name '*.SQL')
   done < <(find $PRODUCT_DIR/dat/lst/sql/mysql/run -type f -name '*.list')
   
   IFS=$TMP_IFS   # revert back to the original IFS
   find . -name '*.bak' | xargs rm -fv
   chmod -v 770 *.sql 
   cd $PRODUCT_DIR
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 

	do_log "DEBUG STOP  doGenerateSQL"
}
# eof func doGenerateSQL

# eof file: src/bash/qto/funcs/generate-sql.func.sh
<file_sep># file: lib/make/clean-install-img-func.mk
# purpose:
# re-usable util make function to build a docker by clearning its cache hashes first
# usage:
# to include it in your Makefile
# include lib/make/clean-install-img-func.mk
# or
# include $(wildcard lib/make/*.mk)
#
# # function usage example:
# install-web-node:
# 	$(call clean-install-img,web-node,80)

include lib/make/demand-var.func.mk

PRODUCT := $(shell basename $$PWD)
product:= $(shell echo `basename $$PWD`|tr '[:upper:]' '[:lower:]')

define clean-install-img
	@clear
	$(call demand-var,ENV)
	docker build . -t ${product}-$(1)-img \
		--no-cache \
		--build-arg UID=$(shell id -u) \
		--build-arg GID=$(shell id -g) \
		--build-arg ENV=$(ENV) \
		--build-arg PRODUCT=${PRODUCT} \
		-f src/docker/$(1)/Dockerfile

	-@docker container stop $$(docker ps -aqf "name=${product}-${1}-con") 2> /dev/null
	-@docker container rm $$(docker ps -aqf "name=${product}-${1}-con") 2> /dev/null

	docker run -it -d --restart=always \
		-v $$(pwd):/opt/${PRODUCT} \
		-v $$HOME/.aws:/home/appusr/.aws \
		-v $$HOME/.ssh:/home/appgrp/.ssh \
		-v $$HOME/.kube:/home/appusr/.kube \
    -p $(2):$(2) \
		--name ${product}-$(1)-con ${product}-${1}-img ;
	@echo -e to attach run: "\ndocker exec -it ${product}-${1}-con /bin/bash"
	@echo -e to get help run: "\ndocker exec -it ${product}-${1}-con ./run --help"
endef

define stop-img-containers
	@clear
	-@docker container stop $$(docker ps -aqf "name=${product}-${1}-con") 2> /dev/null
	-@docker container rm $$(docker ps -aqf "name=${product}-${1}-con") 2> /dev/null
endef
<file_sep>#!/usr/bin/env bash
# usage:
# bash src/tpl/psql-code-generator/psql-code-generator.sh dev_qto release_issues monthly_issues
#
# check also: https://mojolicious.org/perldoc/Mojo/Template


main(){

   do_print=0 ;  # set to 0 to not print a certain function ...
   do_init "$@"
   do_chk_print_usage "$@"
   do_set_vars "$@"
   do_get_psql_meta_data
   do_print=0; do_print_meta_json_str
   do_print=0; do_build_cols_lst_comma
   do_print=1; do_build_cols_lst_table_col
   do_print=0; do_build_cols_lst_old
   do_print=0; do_build_cols_lst_new
   do_print=0; do_build_cols_lst_excluded
   do_print=0; do_generate_psql_html_with_perl_code_example
   do_print=1; do_generate_psql_select_all
   do_print=1; do_run_sql=1; do_generate_psql_select_into_another_table
   do_exit 0
}



do_generate_psql_select_into_another_table(){
   msg='generate the select into another table string'

   IFS='' read -r -d '' perl_code <<"EOF_SELECT_INTO_ANOTHER_TABLE_PERL_CODE"
      use strict; use warnings; binmode STDOUT, ":utf8"; use utf8; use JSON; use Data::Printer;use Mojo::Template; use feature ':5.12';
      my $mt = Mojo::Template->new;
      say $mt->render(<<'EOF', $ENV{'postgres_app_db'}, $ENV{'table_name_01'},$ENV{'table_name_02'} , $ENV{'cols_lst_comma'}, $ENV{'cols_lst_excluded'});
      % my ($postgres_app_db, $table_name_01, $table_name_02, $cols_lst_comma, $cols_lst_excluded) = @_;
      INSERT INTO <%= $table_name_02 %>
      ( <%= $cols_lst_comma %> )
      SELECT <%= $cols_lst_comma %>
      FROM <%= $table_name_01 %>
      ON CONFLICT (id) DO UPDATE SET
      <%= $cols_lst_excluded %>;
EOF

EOF_SELECT_INTO_ANOTHER_TABLE_PERL_CODE

   if [[ $do_print -eq 1 ]]; then
      echo "::: start $msg"
      perl -e "$perl_code" | tee /tmp/$postgres_app_db.select-$table_name_01-into-$table_name_02.sql
      echo -e ":::  stop $msg \n\n"
      if [[ $do_run_sql -eq 1 ]]; then
         PGPASSWORD=${postgres_sys_usr_admin_pw:-} psql -v -t -X -w -U ${postgres_sys_usr_admin:-} \
            --port $postgres_rdbms_port --host $postgres_rdbms_host -t -d ${postgres_app_db:-} \
            < /tmp/$postgres_app_db.select-$table_name_01-into-$table_name_02.sql
      fi
   fi
}


do_generate_psql_select_all(){
   msg='generate the select all string'

   IFS='' read -r -d '' perl_code <<"EOF_SELECT_ALL_PERL_CODE"
      use strict; use warnings; binmode STDOUT, ":utf8"; use utf8; use JSON; use Data::Printer;use Mojo::Template; use feature ':5.12';
      my $mt = Mojo::Template->new;
      say $mt->render(<<'EOF', $ENV{'postgres_app_db'}, $ENV{'table_name_01'}, $ENV{'cols_lst_comma'});
      % my ($postgres_app_db, $table_name_01, $cols_lst_comma) = @_;
      % say "SELECT $cols_lst_comma FROM $postgres_app_db.$table_name_01;"
EOF
EOF_SELECT_ALL_PERL_CODE

   if [[ $do_print -eq 1 ]]; then
      echo "::: start $msg"
      perl -e "$perl_code"
      echo -e ":::  stop $msg \n\n"
   fi
}

# and this is how your generate code in 3 run-times - bash , perl , and Mojo::Template
do_generate_psql_html_with_perl_code_example(){
   msg='generate the perl code'

   IFS='' read -r -d '' perl_code <<"EOF_PERL_CODE"
      use strict; use warnings; binmode STDOUT, ":utf8"; use utf8; use JSON; use Data::Printer;use Mojo::Template; use feature ':5.12';
      my $mt = Mojo::Template->new;
      my $json_var = decode_json($ENV{'meta_json_str'}) ;
      p $json_var ;
      say $mt->render(<<'EOF', [1 .. 13], 'Hello World!');
      % my ($numbers, $title) = @_;
      <div>
        <h1><%= $title %></h1>
          % for my $i (@$numbers) {
              Test <%= $i %>
          % }
      </div>
EOF
EOF_PERL_CODE

   if [[ $do_print -eq 1 ]]; then
      echo "::: start $msg"
      perl -e "$perl_code"
      echo -e ":::  stop $msg \n\n"
   fi
}


#------------------------------------------------------------------------------
# init the minimal possible amount of vars to even fail properly
#------------------------------------------------------------------------------
do_init(){
   umask 022  ; set -u -o pipefail
   call_start_dir=`pwd`
   run_unit_bash_dir=$(perl -e 'use File::Basename; use Cwd "abs_path"; print dirname(abs_path(@ARGV[0]));' -- "$0")
   tmp_dir="$run_unit_bash_dir/tmp/.tmp.$$"
   mkdir -p "$tmp_dir"
   ( set -o posix ; set )| sort >"$tmp_dir/vars.before"
   my_name_ext=`basename $0`
   RUN_UNIT=${my_name_ext%.*}

   export postgres_app_db=${1:-}
   export table_name_01=${2:-}
   export table_name_02=${3:-}
}

#------------------------------------------------------------------------------
# register and show the run-time vars
#------------------------------------------------------------------------------
do_set_vars(){

   cd $run_unit_bash_dir
   for i in {1..3} ; do cd .. ; done ; export PRODUCT_DIR=`pwd`;
   environment_name=$(basename "$PRODUCT_DIR")
   cd $PRODUCT_DIR

   source $PRODUCT_DIR/.env
   #
   source $PRODUCT_DIR/lib/bash/funcs/export-json-section-vars.sh

   test -z "${PROJ_INSTANCE_DIR-}" && export PROJ_INSTANCE_DIR="$PRODUCT_DIR"
   test -z ${PROJ_CONF_FILE:-} && export PROJ_CONF_FILE="$PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json"

   if [ "$environment_name" == "$RUN_UNIT" ]; then
      product_dir=$PRODUCT_DIR
   else
      cd .. ; product_dir=`pwd`;
   fi

   cd .. ; product_base_dir=`pwd`; org_name=$(basename `pwd`)

   do_export_json_section_vars $PROJ_CONF_FILE '.env.db'

   if [[ $do_print -eq 1 ]]; then
      ( set -o posix ; set ) | sort >"$tmp_dir/vars.after"
      echo "INFO using the following vars:"
      cmd="$(comm -3 $tmp_dir/vars.before $tmp_dir/vars.after | perl -ne 's#\s+##g;print "\n $_ "' )"
      echo -e "$cmd"
      do_flush_screen
   fi

}


do_chk_print_usage(){

   db=dev_qto
   if [[ -z "${1:-}" || -z "${2:-}" ]]; then
      cat << EOF_USAGE

      psql-code-generator - a script for generating code
      bash $0 <<db-name>> <<table-name>>
      example:
      bash $0 dev_qto release_issues

      example to generate code for all the tables in a db:
      while read -r t; do $0 dev_qto \$t ; done < <(psql -d dev_qto -t -q -c "SELECT table_name FROM information_schema.tables where 1=1 and table_catalog='dev_qto' and table_schema='public'")

      needless to say anything you print you could append to a file as follows:
      <<command>> | tee -a /tmp/generated-code.sql

EOF_USAGE

   do_exit 1
   fi
}


do_get_psql_meta_data(){
   export meta_json_str=$(PGPASSWORD=${postgres_sys_usr_admin_pw:-} psql -v -t -X -w -U ${postgres_sys_usr_admin:-} --port $postgres_rdbms_port --host $postgres_rdbms_host -t -d ${postgres_app_db:-} -c "select array_to_json(array_agg(row_to_json(t))) from ( SELECT * from information_schema.columns where 1=1 and table_catalog='"$postgres_app_db"' and table_name='"$table_name_01"') t")
}


do_print_maybe(){
   msg="$1" ; shift
   to_print="$*"
   test $do_print -eq 1 && echo "::: start $msg" && echo $to_print && echo -e "::: stop  $msg \n\n"
}


do_print_meta_json_str(){
   msg='the whole json string'
   test $do_print -eq 1 && echo "::: start $msg"
   test $do_print -eq 1 && echo $meta_json_str|jq -r '.[]'
   test $do_print -eq 1 && echo -e ":::  stop $msg \n\n"
}


do_build_cols_lst_comma(){
   msg='the columns only'
   while read -r c ; do cols_lst_comma="${cols_lst_comma:-} , $c" ; done < <(echo $meta_json_str|jq -r '.[]|.column_name')
   export cols_lst_comma=$(echo $cols_lst_comma|cut -c 3-)
   do_print_maybe "$msg" $cols_lst_comma
}

do_build_cols_lst_table_col(){
   msg='the tables and columns only'
   while read -r c ; do cols_lst_table_col="${cols_lst_table_col:-} , $table_name_01.$c" ; done < <(echo $meta_json_str|jq -r '.[]|.column_name')
   export cols_lst_table_col=$(echo $cols_lst_table_col|cut -c 3-)
   do_print_maybe "$msg" $cols_lst_table_col
}


do_build_cols_lst_old(){
   msg='the OLD columns list '
   while read -r c ; do cols_lst_old="${cols_lst_old:-}, OLD.$c" ; done < <(echo $meta_json_str|jq -r '.[]|.column_name')
   cols_lst_old=$(echo $cols_lst_old|cut -c 3-)
   do_print_maybe "$msg" $cols_lst_old
}


do_build_cols_lst_new(){
   msg='the NEW columns list'
   while read -r c ; do cols_lst_new="${cols_lst_new:-}, NEW.$c" ; done < <(echo $meta_json_str|jq -r '.[]|.column_name')
   export cols_lst_new=$(echo $cols_lst_new|cut -c 3-)
   do_print_maybe "$msg" $cols_lst_new
}


do_build_cols_lst_excluded(){
   msg='the excluded columns list '
   while read -r c ; do cols_lst_excluded="${cols_lst_excluded:-}, $c = excluded.$c" ; done < <(echo $meta_json_str|jq -r '.[]|.column_name')
   export cols_lst_excluded=$(echo $cols_lst_excluded|cut -c 3-)
   do_print_maybe "$msg" $cols_lst_excluded
}


do_exit(){
   exit_code="$1";shift
   rm -rf "$run_unit_bash_dir/tmp" #clear the tmpdir
   cd $call_start_dir
   exit $exit_code
}


main "$@"
<file_sep>CREATE TABLE meta_cols as
         SELECT DISTINCT
             ROW_NUMBER () OVER (ORDER BY pgc.relname , a.attnum) as id,gen_random_uuid() as guid
             , pgc.relname as table_name
             , a.attnum as attribute_number
             , a.attname as attribute_name
             , format_type(a.atttypid, a.atttypmod) as data_type
             , a.atttypmod as char_max_length
             , a.attnotnull as not_null
             , com.description as comment
             , coalesce(i.indisprimary,false) as is_primary_key
         FROM pg_attribute a
         JOIN pg_class pgc ON pgc.oid = a.attrelid
         LEFT JOIN pg_index i ON
             (pgc.oid = i.indrelid AND i.indkey[0] = a.attnum)
         LEFT JOIN pg_description com on
             (pgc.oid = com.objoid AND a.attnum = com.objsubid)
         LEFT JOIN pg_attrdef def ON
             (a.attrelid = def.adrelid AND a.attnum = def.adnum)
         LEFT JOIN pg_catalog.pg_namespace n ON n.oid = pgc.relnamespace
         WHERE 1=1
            AND pgc.relkind IN ('r','')
             AND n.nspname <> 'pg_catalog'
             AND n.nspname <> 'information_schema'
             AND n.nspname !~ '^pg_toast'
         AND a.attnum > 0 AND pgc.oid = a.attrelid
         AND pg_table_is_visible(pgc.oid)
         AND NOT a.attisdropped
         ORDER BY id
;

<file_sep>do_chk_provision_ssh_keys(){

   do_export_json_section_vars $product_dir/cnf/env/$ENV.env.json '.env.db'
   which expect || sudo apt-get update && sudo apt-get install -y expect

   # if the ssh key does not exist create it ...
   # the perl code for building the private key file path is :
   # my $private_key_fpath   = $ENV{"HOME"} . '/.ssh/id_rsa.qto.' . $ENV ; 
   prv_key_fpath=$HOME/.ssh/id_rsa.qto.$ENV
   test -f $prv_key_fpath || {
   expect <<- EOF_EXPECT
      set timeout -1
      spawn ssh-keygen -t rsa -b 4096 -C $AdminEmail -f $prv_key_fpath -N ''
      expect "Enter passphrase (empty for no passphrase): "
      send -- "\r"
		expect "Enter same passphrase again: "
      send -- "\r"
      expect eof
EOF_EXPECT

      echo created the fillowing private key file : $prv_key_fpath
   }

   for ENV in `echo dev tst prd`; do 
      jwt_private_key_file=~/.ssh/qto.$ENV.jwtRS256.key
      jwt_public_key_file=~/.ssh/qto.$ENV.jwtRS256.key.pub
      echo << EOF_USING
      creating and using the following private and public key files for the Guardian's JWT's :
      $jwt_private_key_file
      $jwt_public_key_file
EOF_USING
      
      test -f $jwt_private_key_file || ssh-keygen -t rsa -b 4096 -m PEM -f $jwt_private_key_file -N ''
      test -f $jwt_public_key_file && rm -v $jwt_public_key_file
      openssl rsa -in $jwt_private_key_file -pubout -outform PEM -out $jwt_public_key_file
   done;

}
<file_sep>common / shared configuration files for your qto apps should be placed here ... as well as
backuped hosts configuration files below their host dirs
<file_sep># vim:set ft=dockerfile:
FROM ubuntu:18.04

ARG TZ=$TZ
ARG ENV=$ENV
ARG USER=$USER
ARG UID=$UID
ARG GROUP=$GROUP
ARG GID=$GID
ARG PRODUCT_DIR=$PRODUCT_DIR
ARG postgres_app_db=$postgres_app_db
ARG postgres_sys_usr_admin=$postgres_sys_usr_admin
ARG root_pwd=$<PASSWORD>
ARG app_user_pwd=$<PASSWORD>

# obs !!! todo: parametrize qto-190616104728
RUN echo "root:$root_pwd" | chpasswd
RUN echo 'export PS1="`date "+%F %T"` \u@\h  \w \\n\\n  "' >> /root/.bashrc

RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone 
# install the most basic binaries
RUN apt-get update && apt-get install -y wget curl sudo perl zip unzip gnupg2 gnupg1 git bash jq vim

# start :: add the application user
RUN set -x ; addgroup --gid "$GID" "$GROUP" && \
    adduser \
		 --gid $GID \
		 --shell "/bin/bash" \
		 --home "/home/$USER" \
		 --uid $UID \
		 "$USER" && exit 0 ; exit 1

RUN echo "$USER"':'"$app_user_pwd" | chpasswd
RUN echo "$USER ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
# stop :: add the application user


# install support binaries 
RUN apt-get clean && apt-get update && apt-get install -y nmap perl net-tools exuberant-ctags mutt curl wget libwww-curl-perl libdbd-pgsql libxml-atom-perl libxml-atom-perl tar gzip graphviz python-setuptools python-dev build-essential gpgsm nodejs lsof libssl1.0-dev procps node-gyp nodejs-dev npm awscli

#
# start ::: install postgres 
# Add the PostgreSQL PGP key to verify their Debian packages.
# It should be the same key as https://www.postgresql.org/media/keys/ACCC4CF8.asc
RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -

# add the PostgreSQL's repository
RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main"' \
   > /etc/apt/sources.list.d/pgdg.list

# add the most basic binaries
RUN apt-get clean && apt-get update && apt-get install -f -y postgresql-server-dev-11 postgresql-client-11 postgresql-contrib-11

## ensure the postresql starts on boot 
RUN sudo update-rc.d postgresql enable

RUN mkdir -p /etc/postgresql/11/main/ && mkdir -p /var/lib/postgresql/11/main

RUN echo "postgres:postgres" | chpasswd
RUN echo 'export PS1="`date "+%F %T"` \u@\h  \w \\n\\n  "' >> /var/lib/postgresql/.bashrc


## add the uuid generation capability enabling extensions
RUN /etc/init.d/postgresql restart && \
sudo -u postgres psql -c "CREATE USER usrqtoadmin WITH SUPERUSER CREATEROLE CREATEDB REPLICATION BYPASSRLS PASSWORD '<PASSWORD>';" && \
sudo -u postgres psql -c "grant all privileges on database postgres to usrqtoadmin ;" && \
sudo -u postgres psql template1 -c 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp";' && \
sudo -u postgres psql template1 -c 'CREATE EXTENSION IF NOT EXISTS "pgcrypto";' && \
sudo -u postgres psql template1 -c 'CREATE EXTENSION IF NOT EXISTS "dblink";' 

ENV psql_cnf_dir='/etc/postgresql/11/main'
RUN cp -v $psql_cnf_dir/pg_hba.conf $psql_cnf_dir/pg_hba.conf.orig.bak && \
   cp -v $PRODUCT_DIR/cnf/postgres/$psql_cnf_dir/pg_hba.conf $psql_cnf_dir/pg_hba.conf && \
   chown postgres:postres $psql_cnf_dir

RUN sudo cp -v $psql_cnf_dir/postgresql.conf $psql_cnf_dir/postgresql.conf.orig && \
   sudo cp -v $PRODUCT_DIR/cnf/postgres/$psql_cnf_dir/postgresql.conf $psql_cnf_dir/postgresql.conf

RUN chown -R postgres:postgres "/etc/postgresql" && \
    chown -R postgres:postgres "/var/lib/postgresql" && \
    chown -R postgres:postgres "/etc/postgresql/11/main/pg_hba.conf" && \
    chown -R postgres:postgres "/etc/postgresql/11/main/postgresql.conf"

# add VOLUMEs to allow backup of config, logs and databases
# VOLUME  ["/etc/postgresql", "/var/log/postgresql", "/var/lib/postgresql"]

# :: stop install postgres 

# :: start install perl modules
RUN apt-get update && apt-get install -y cpanminus && apt-get install -y liblocal-lib-perl

USER $USER
# set the default command to run when starting the container
#RUN perl -MCPAN -Mlocal::lib -e 'install Data::Printer' && \
#   perl -MCPAN -Mlocal::lib -e 'install Term::Prompt' && \
#   perl -MCPAN -Mlocal::lib -e 'install Email::Valid' && \
#   perl -MCPAN -Mlocal::lib -e 'install Excel::Writer::XLSX' && \
#   perl -MCPAN -Mlocal::lib -e 'install Spreadsheet::ParseExcel' && \
#   perl -MCPAN -Mlocal::lib -e 'install Spreadsheet::XLSX' && \
#   perl -MCPAN -Mlocal::lib -e 'install DBD::Pg' && \
#   perl -MCPAN -Mlocal::lib -e 'install Tie::Hash::DBD' && \
#   perl -MCPAN -Mlocal::lib -e 'install Text::CSV_XS' && \
#   perl -MCPAN -Mlocal::lib -e 'install File::Copy::Recursive' && \
#   perl -MCPAN -Mlocal::lib -e 'install Test::Trap' && \
#   perl -MCPAN -Mlocal::lib -e 'install Test::Most' && \
#   perl -MCPAN -Mlocal::lib -e 'install Tie::Hash::DBD' && \
#   perl -MCPAN -Mlocal::lib -e 'install Scalar::Util::Numeric' && \
#   perl -MCPAN -Mlocal::lib -e 'install IPC::System::Simple' && \
#   perl -MCPAN -Mlocal::lib -e 'install Mojo::Pg' && \
#   perl -MCPAN -Mlocal::lib -e 'install Mojolicious::Plugin::BasicAuthPlus' && \
#   perl -MCPAN -Mlocal::lib -e 'install Mojolicious::Plugin::StaticCache' && \
#   perl -MCPAN -Mlocal::lib -e 'install Mojolicious::Plugin::RenderFile' && \
#   perl -MCPAN -Mlocal::lib -e 'install Mojolicious::Plugin::Authentication' && \
#   perl -MCPAN -Mlocal::lib -e 'install Mojo::JWT' && \
#   perl -MCPAN -Mlocal::lib -e 'install Authen::Passphrase::BlowfishCrypt' && \
#   perl -MCPAN -Mlocal::lib -e 'install Net::Google::DataAPI::Auth::OAuth2' && \
#   perl -MCPAN -Mlocal::lib -e 'CPAN::Shell->force(qw( install Net::Google::DataAPI::Auth::OAuth2));' && \
#   perl -MCPAN -Mlocal::lib -e 'install Net::Google::Spreadsheets::V4' && \
#   perl -MCPAN -Mlocal::lib -e 'install Net::Google::Spreadsheets;' && \
#   perl -MCPAN -Mlocal::lib -e 'install DBIx::ProcedureCall' && \
#   perl -MCPAN -Mlocal::lib -e 'install JSON::Parse' 

# stop ::: install perl modules

USER $USER
WORKDIR $PRODUCT_DIR

# CMD ["bash","-c","while true; do sleep 1; done;"]
CMD bash -c $PRODUCT_DIR/src/bash/qto/install/docker/docker-entry-point.sh
<file_sep>doSpawnAwsEc2(){

   #
   test -z ${PROJ_CONF_FILE:-} && export PROJ_CONF_FILE="$PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json"

   do_export_json_section_vars $PROJ_CONF_FILE '.env.aws'
   mkdir -p $PROJ_INSTANCE_DIR/src/terraform/qto

   test -f ~/.aws/credentials || \
      do_exit 1 "run the aws configure command - did not find the ~/.aws/credentials file !!!"
   export AWS_ACCESS_KEY_ID=$(cat ~/.aws/credentials| grep -i aws_access_key_id | awk '{print $3}')
   export AWS_SECRET_ACCESS_KEY=$(cat ~/.aws/credentials| grep -i AWS_SECRET_ACCESS_KEY| awk '{print $3}')

   sudo timedatectl set-ntp no; timedatectl
   sudo service ntp stop ; sudo service ntp start; ntpq -p

   export key_name=$(basename `eval echo ${private_ssh_key_fpath:-}`)
   export public_ssh_key_fpath=$(eval echo ${public_ssh_key_fpath:-})
   export public_ssh_key_content=$(cat ${public_ssh_key_fpath:-})

	main_tf_file="$PROJ_INSTANCE_DIR/src/terraform/qto/main.tf"
	test -f "$main_tf_file" && rm -v "$main_tf_file"

   mkdir -p "$PROJ_INSTANCE_DIR/src/terraform/qto"
   rm -r $PROJ_INSTANCE_DIR/src/terraform/qto/*
   python3 "$PROJ_INSTANCE_DIR/src/python/tpl_gen.py"
   perl -pi -e 'foreach $key(sort keys %ENV){ s|\$$key|$ENV{$key}|g}' "$main_tf_file"
   #perl -e 'foreach $key(sort keys %ENV){ print "echo \$$key key:$key \n, val:$ENV{$key} \n"}'

   cd $PROJ_INSTANCE_DIR/src/terraform/qto/
   terraform init ; terraform plan ; test $? -ne 0 && do_exit 1 "terraform plan failed"
	terraform apply -auto-approve

   rm -v $PROJ_INSTANCE_DIR/src/terraform/qto/main.tf
}
<file_sep>#------------------------------------------------------------------------------
# usage example:
# source $PRODUCT/lib/bash/funcs/read-conf-section.sh
# do_read_conf_section '.env.db'
#------------------------------------------------------------------------------
do_read_conf_section(){

   PROJ_CONF_FILE=$1; shift

   test -f ${PROJ_CONF_FILE:-} || {
      echo "ERROR : the json_file: $PROJ_CONF_FILE does not exist !!! Nothing to do" && exit 1
   }

   section="$1"
   test -z "$section" && \
      echo "ERROR : the section in do_read_conf_section is empty !!! Nothing to do !!!" && exit 1

   do_flush_screen

   echo "INFO : exporting vars from cnf ${PROJ_CONF_FILE:-}: "
   while read -r l ; do
      key=$(echo $l|cut -d'=' -f1)
      val=$(echo $l|cut -d'=' -f2)
      eval "$(echo -e 'export '$key=\"\"$val\"\")"
      echo "INFO : $key=$val"
   done < <(cat $PROJ_CONF_FILE| jq -r "$section"'|keys_unsorted[] as $key|"\($key)=\"\(.[$key])\""')
}
<file_sep>do_chk_install_nginx(){

	if [ ! "grep -q nginx /etc/apt/sources.list" ] ;  # check to avoid adding lines multiple times, if this record already exists
	then
		printf "\nAdding nginx repositories to install the latest nginx version.\n\n"
		set -x
		sudo bash -c 'cat >> /etc/apt/sources.list << EOF_NGINX_REPOS
# nginx repos
deb https://nginx.org/packages/ubuntu/ bionic nginx
deb-src https://nginx.org/packages/ubuntu/ bionic nginx
EOF_NGINX_REPOS'

		wget http://nginx.org/keys/nginx_signing.key
		sudo apt-key add nginx_signing.key
		sudo apt-get update
	fi

   test -f /etc/nginx/nginx.conf || sudo apt-get install -y nginx
   test -f /etc/nginx/nginx.conf && sudo cp -v /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak

  

}
<file_sep>#!/bin/bash

set -x

test -z ${PRODUCT:-} && PRODUCT=qto
test -z ${APPUSER:-} && APPUSER=appusr


trap : TERM INT; sleep infinity & wait
<file_sep>#------------------------------------------------------------------------------
# usage example:
# source $PRODUCT_DIR/lib/bash/funcs/flush-screen.sh
# do_flush_screen
#
# if you have sourced $PRODUCT_DIR/lib/bash/funcs/export-json-section-vars.sh before,
# then this function is also imported, so sourcing again is not necessary
#------------------------------------------------------------------------------
do_flush_screen(){
	printf "\033[2J";printf "\033[0;0H"
}<file_sep># vim:set ft=dockerfile:
FROM ubuntu:18.04

USER root

ARG PRODUCT_DIR=/opt/csitea/qto/qto.0.6.5.dev.usrqtoadmin
ENV PRODUCT_DIR $PRODUCT_DIR
ARG host_host_name
ENV host_host_name $host_host_name

WORKDIR /opt/
ADD . $PRODUCT_DIR
WORKDIR $PRODUCT_DIR

# obs !!! todo: parametrize qto-190616104728
RUN echo "root:root" | chpasswd
RUN echo 'export PS1="`date "+%F %T"` \u@\h  \w \\n\\n  "' >> /root/.bashrc
# src: https://stackoverflow.com/a/28769950/65706

RUN chmod 755 $PRODUCT_DIR/src/bash/qto/install/docker/install-test.sh && \
   bash $PRODUCT_DIR/src/bash/qto/install/docker/install-test.sh

CMD ["bash","-c","while true; do sleep 1; done;"]
<file_sep># src/bash/qto/funcs/remove-package.test.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doTestRemovePackage comments ...
# ---------------------------------------------------------
doTestRemovePackage(){

	do_log "DEBUG START doTestRemovePackage"
	
	cat doc/txt/qto/tmpl/remove-package.test.txt
	sleep 2
	# add your action implementation code here ... 

	do_log "DEBUG STOP  doTestRemovePackage"
}
# eof func doTestRemovePackage


# eof file: src/bash/qto/funcs/remove-package.test.sh
<file_sep>         // usage: clear ; cd src/js/node/js-unit-tests/01-set-url-param ; npm install ; cd -
         // usage: clear ; cd src/js/node/js-unit-tests/01-set-url-param ; npm test ; cd -
         // prereqs: , nodejs , mocha
         // URI = scheme:[//authority]path[?paramName1=paramValue1&paramName2=paramValue2][#fragment]
         // call by: uri = uri.setUriParam("as","md")
         String.prototype.setUriParam = function (paramName, paramValue) {
            var uri = this
            var fragment = ( uri.indexOf('#') === -1 ) ? '' : uri.split('#')[1] 
            uri = ( uri.indexOf('#') === -1 ) ? uri : uri.split('#')[0] 
            if ( uri.indexOf("?") === -1 ) { uri = uri + '?&' } 
            uri = uri.replace ( '?' + paramName , '?&' + paramName)
            var toRepl = (paramValue != null) ? ('$1' + paramValue) : ''
            var toSrch = new RegExp('([&]' + paramName + '=)(([^&#]*)?)')
            uri = uri.replace(toSrch,toRepl)
            if (uri.indexOf(paramName + '=') === -1 && toRepl != '' ) {
               var ampersandMayBe = uri.endsWith('&') ? '' : '&'
               uri = uri + ampersandMayBe + paramName + "=" + String(paramValue)
            }
            uri = ( fragment.length == 0 ) ? uri : (uri+"#"+fragment) //may-be re-add the fragment 
            return uri
         }

         var assert = require('assert');
         describe('replacing url param value', function () {
            
            // scheme://authority/path[?p1=v1&p2=v2#fragment
            // a clean url
            it('http://org.com/path -> http://org.com/path?&prm=tgt_v', function (){
               var uri = 'http://site.eu:80/qto/view/devops_guide_doc'
               var uriExpected = 'http://site.eu:80/qto/view/devops_guide_doc?&bid=10'
               var uriActual = uri.setUriParam("bid",10)
               assert.equal(uriActual, uriExpected);
            });
           
            // has the url param existing after the ? with num value
            it('http://org.com/path?prm=src_v -> http://org.com/path?&prm=tgt_v', function (){
               var uri = 'http://site.eu:80/qto/view/devops_guide_doc?bid=57'
               var uriExpected = 'http://site.eu:80/qto/view/devops_guide_doc?&bid=10'
               var uriActual = uri.setUriParam("bid",10)
               assert.equal(uriActual, uriExpected);
            });
            
            // has the url param existing after the ? but string value
            it('http://org.com/path?prm=src_v -> http://org.com/path?&prm=tgt_v', function (){
               var uri = 'http://site.eu:80/qto/view/devops_guide_doc?bid=boo-bar'
               var uriExpected = 'http://site.eu:80/qto/view/devops_guide_doc?&bid=boo-bar-baz'
               var uriActual = uri.setUriParam("bid","boo-bar-baz")
               assert.equal(uriActual, uriExpected);
            });

            // has the url param existing after the ?& but string value
            it('http://org.com/path?&prm=src_v -> http://org.com/path?&prm=tgt_v', function (){
               var uri = 'http://site.eu:80/qto/view/devops_guide_doc?&bid=5'
               var uriExpected = 'http://site.eu:80/qto/view/devops_guide_doc?&bid=10'
               var uriActual = uri.setUriParam("bid",10)
               assert.equal(uriActual, uriExpected);
            });
            
            // has the url param existing after the ? with other param
            it('http://org.com/path?prm=src_v&other_p=other_v -> http://org.com/path?&prm=tgt_v&other_p=other_v', function (){
               var uri = 'http://site.eu:80/qto/view/devops_guide_doc?bid=5&other_p=other_v'
               var uriExpected = 'http://site.eu:80/qto/view/devops_guide_doc?&bid=10&other_p=other_v'
               var uriActual = uri.setUriParam("bid",10)
               assert.equal(uriActual, uriExpected);
            });

            // has the url param existing after the ?& with other param
            it('http://org.com/path?&prm=src_v&other_p=other_v -> http://org.com/path?&prm=tgt_v&other_p=other_v', function (){
               var uri = 'http://site.eu:80/qto/view/devops_guide_doc?&bid=5&other_p&other_v'
               var uriExpected = 'http://site.eu:80/qto/view/devops_guide_doc?&bid=10&other_p&other_v'
               var uriActual = uri.setUriParam("bid",10)
               assert.equal(uriActual, uriExpected);
            });

            // has the url param existing after the ? with other param with fragment
            it('http://org.com/path?prm=src_v&other_p=other_v#f -> http://org.com/path?&prm=tgt_v&other_p=other_v#f', function (){
               var uri = 'http://site.eu:80/qto/view/devops_guide_doc?bid=5&other_p=other_v#f'
               var uriExpected = 'http://site.eu:80/qto/view/devops_guide_doc?&bid=10&other_p=other_v#f'
               var uriActual = uri.setUriParam("bid",10)
               assert.equal(uriActual, uriExpected);
            });

            // has the url param existing after the ?& with other param with fragment
            it('http://org.com/path?&prm=src_v&other_p=other_v#f -> http://org.com/path?&prm=tgt_v&other_p=other_v#f', function (){
               var uri = 'http://site.eu:80/qto/view/devops_guide_doc?&bid=5&other_p&other_v#f'
               var uriExpected = 'http://site.eu:80/qto/view/devops_guide_doc?&bid=10&other_p&other_v#f'
               var uriActual = uri.setUriParam("bid",10)
               assert.equal(uriActual, uriExpected);
            });

            // remove the param-name , param-value pair
            it('http://org.com/path?prm=src_v&other_p=other_v#f -> http://org.com/path?&prm=tgt_v&other_p=other_v#f', function (){
               var uri = 'http://site.eu:80/qto/view/devops_guide_doc?bid=5&other_p=other_v#f'
               var uriExpected = 'http://site.eu:80/qto/view/devops_guide_doc?&other_p=other_v#f'
               var uriActual = uri.setUriParam("bid",null)
               assert.equal(uriActual, uriExpected);
            });

            // remove the param-name , param-value pair
            it('http://org.com/path?&prm=src_v&other_p=other_v#f -> http://org.com/path?&prm=tgt_v&other_p=other_v#f', function (){
               var uri = 'http://site.eu:80/qto/view/devops_guide_doc?&bid=5&other_p=other_v#f'
               var uriExpected = 'http://site.eu:80/qto/view/devops_guide_doc?&other_p=other_v#f'
               var uriActual = uri.setUriParam("bid",null)
               assert.equal(uriActual, uriExpected);
            });
            
            // add a new param name , param value pair
            it('http://org.com/path?prm=src_v&other_p=other_v#f -> http://org.com/path?&prm=tgt_v&other_p=other_v#f', function (){
               var uri = 'http://site.eu:80/qto/view/devops_guide_doc?&other_p=other_v#f'
               var uriExpected = 'http://site.eu:80/qto/view/devops_guide_doc?&other_p=other_v&bid=foo-bar#f'
               var uriActual = uri.setUriParam("bid","foo-bar")
               assert.equal(uriActual, uriExpected);
            });

         });
         // todo:require('./lib/file1.js');
<file_sep>doProvisionHttps(){
   do_export_json_section_vars $PRODUCT_DIR/cnf/env/$ENV.env.json '.env.app'

   sudo test -f /etc/letsencrypt/live/$web_host/privkey.pem && {
      source $PRODUCT_DIR/lib/bash/funcs/export-json-section-vars.sh
      for env in `echo dev tst prd`; do \
         do_export_json_section_vars $PRODUCT_DIR/cnf/env/$env.env.json '.env.app'
         sudo cp -v "$PRODUCT_DIR"'/cnf/nginx/etc/nginx/sites-available/%env%.https-site.conf' \
            "/etc/nginx/sites-available/$env.https-site.conf"
         sudo perl -pi -e 's|\%nginx_port\%|'"$nginx_port"'|g' "/etc/nginx/sites-available/$env.https-site.conf"
         sudo perl -pi -e 's|\%nginx_port\%|'"$nginx_port"'|g' "/etc/nginx/sites-available/$env.http-site.conf"
         sudo perl -pi -e 's|\%https_port\%|'"$https_port"'|g' "/etc/nginx/sites-available/$env.https-site.conf"
         sudo perl -pi -e 's|\%https_port\%|'"$https_port"'|g' "/etc/nginx/sites-available/$env.http-site.conf"
         sudo perl -pi -e 's|#uncomment-for-https ||g' "/etc/nginx/sites-available/$env.http-site.conf"
         sudo perl -pi -e 's|\%mojo_hypnotoad_port\%|'"$mojo_hypnotoad_port"'|g' "/etc/nginx/sites-available/$env.https-site.conf"
         sudo perl -pi -e 's|\%web_host\%|'"$web_host"'|g' "/etc/nginx/sites-available/$env.https-site.conf"
         sudo perl -pi -e 's|\%port\%|'"$port"'|g' "/etc/nginx/sites-available/$env.https-site.conf"
         sudo perl -pi -e 's|#todo-https-me||g' "/etc/nginx/sites-available/$env.http-site.conf"
         
         sudo ln -fs /etc/nginx/sites-available/$env.https-site.conf /etc/nginx/sites-enabled/$env.https-site.conf
         sudo ls -la /etc/nginx/sites-enabled/$env.https-site.conf
      done ;

      sudo chown -R www-data:www-data /etc/nginx
      sudo service nginx restart
      sudo service nginx status
      do_export_json_section_vars $PRODUCT_DIR/cnf/env/$ENV.env.json '.env.app'
   }
}
<file_sep>#!/bin/bash

do_chk_provision_db_users(){

   test -z ${PROJ_CONF_FILE:-} && export PROJ_CONF_FILE="$PRODUCT_DIR/cnf/env/$ENV.env.json"
   source $PRODUCT_DIR/lib/bash/funcs/export-json-section-vars.sh
   do_export_json_section_vars $PROJ_CONF_FILE '.env.db'

	# generate the sql for the provisioning of the postgres_sys_usr_admin
	cat << EOF_SQL_CODE > /tmp/$$.tmp.sql
		DO
		\$\$
		DECLARE r record;
			BEGIN
				IF NOT EXISTS (
					SELECT
					FROM   pg_catalog.pg_roles
					WHERE  rolname = '$postgres_sys_usr_admin') THEN
						CREATE ROLE "$postgres_sys_usr_admin" WITH SUPERUSER CREATEROLE
						CREATEDB REPLICATION BYPASSRLS PASSWORD '$<PASSWORD>' LOGIN ;
				END IF;
			END ;
		\$\$;
		ALTER ROLE $postgres_sys_usr_admin WITH SUPERUSER CREATEROLE
			CREATEDB REPLICATION BYPASSRLS PASSWORD  '$<PASSWORD>' LOGIN ;
   	GRANT ALL PRIVILEGES ON DATABASE POSTGRES TO $postgres_sys_usr_admin" ;
EOF_SQL_CODE

	# run the sql for the provisioning of the postgres_sys_usr_admin
	sudo -Hiu postgres PGPASSWORD=$<PASSWORD> psql --port $postgres_rdbms_port \
         --host $postgres_rdbms_host -d postgres < /tmp/$$.tmp.sql
	rm -rv /tmp/$$.tmp.sql

	# verify the sql for the provisioning of the postgres_sys_usr_admin
  sudo -Hiu postgres PGPASSWORD=$<PASSWORD> psql --port $postgres_rdbms_port \
         --host $postgres_rdbms_host -d postgres -c '\du'
}
<file_sep>#!/usr/bin/env python3
# usage: python src/python/append-to-docx.py requirements_doc /path/to/file.docx
import sys
import os
import urllib.request, json 
import docx
from docx import Document
from docx.shared import Cm
from docx.enum.text import WD_ALIGN_PARAGRAPH
from io import StringIO



ikey, full_path, table, path, app, db, user, ht_protocol, web_host, web_port = 0, '', '', '', '', '', '', '', '', ''
def main():
    set_vars()
    tableData, met = getTableData(None)
    buildDoc(table,tableData)
    sys.exit()


def set_vars():
    try:
        global full_path, table, path, app, db, user, ht_protocol, web_host, web_port
        table = sys.argv[1]
        full_path = str(sys.argv[2] or '.')
        app = os.environ['postgres_app_db']
        db = os.environ['postgres_app_db']
        user= os.environ['postgres_app_usr']
        ht_protocol = os.environ['ht_protocol']
        web_host= os.environ['web_host']
        web_port = os.environ['port']
    except(IndexError) as error:
        print (error) 
        traceback.print_stack()
        sys.exit(1)

    return


def getTableData(url):
       
    if ( url == None):
        url = ht_protocol + "://" + web_host + ":" + web_port + "/" + app + "/hiselect/" + table + "?pg-size=10000&bid=0"
    try:
        with urllib.request.urlopen(url) as url:
            data = json.loads(url.read().decode())
            # print(data['dat'])
            print(data['met']['meta_cols'])
            return data['dat'], data['met']['meta_cols']
    except (Exception) as error:
        print(error)


def buildDoc(table,tableData):
    doc = Document(full_path) # open an existing document with existing styles
    #debug print('Loaded {}'.format(tableData))
    for row in tableData:
        # print ('row {}'.format(row))
        level = row['level']
        if ( level == 0 ):
            continue
        if ( level == 1 ):
            levelStyle = 'style-kone-blue-heading-01'
        else: 
            levelStyle = 'Heading ' + str(level)
        title = row['name']
        heading = doc.add_heading( title , level)
        heading.style = doc.styles[levelStyle]
        p = doc.add_paragraph(row['description'])
        if row['src']:
            src = doc.add_paragraph(row['src'])
            src.style = doc.styles['Body Text 3']
        if ( row['formats'] ):
            if (not( str(row['formats']).strip() == "")):
                print ("formats: " + row['formats'])
                formats = json.loads(row['formats'])
                listing_url = formats['listing-url']
                print ("listing_url: " + listing_url)
                listingData , met = getTableData(listing_url)
                #for style in doc.styles:
                    #print ( style )
                global ikey
                doc_table = doc.add_table(1, (len(listingData[0].keys())-2), doc.styles['Table Grid'])
                for lrow in listingData:
                    mikey = -1
                    ikey = 0
                    for mlrowkey in sorted (met.keys()):
                        mlrow = met[str(mlrowkey)]
                        key = mlrow['attribute_name']
                        mikey = mikey + 1
                        if ( not ( key in lrow.keys() ) ):
                            continue
                        if ( not ( key == 'id' or key == 'guid' )): 
                            if ( ikey == 0 ):
                                cells = doc_table.add_row().cells
                                thkey = 0
                                for th in sorted (met.keys()): # this is the title row
                                    mthrow = met[str(th)]
                                    ky = mthrow['attribute_name']
                                    print ( "lrow.keys:")
                                    print ( "ky: " + str(ky))
                                    print(lrow.keys())
                                    if ( (not ( ky in lrow.keys() )) or ky == 'id' or ky == 'guid'):
                                        continue
                                    else:
                                        cells[thkey].text = str(ky)
                                        thkey=thkey+1
                            #print ( str(lrow[key]) ) # the value cells[0].text = lrow[key]
                            cells[ikey].text = str(lrow[key])
                            ikey = ikey + 1
        if row['img_relative_path']:
            ip = doc.add_paragraph()
            r = ip.add_run()
            r.add_text(row['img_name'])
            r.add_text("\n")
            r.add_picture(row['img_relative_path'], width = Cm(15.0))

    doc.save(full_path)
    return 0


main()
<file_sep>-- file: src/sql/pgsql/list-table-priviledges.sql
-- usage: 
-- alias psql="PGPASSWORD=${postgres_sys_usr_admin_pw:-} psql -v -q -t -X -w -U ${postgres_sys_usr_admin:-}"
-- psql -d dev_qto < src/sql/pgsql/list-table-priviledges.sql | less
SELECT grantee, table_name , privilege_type
FROM information_schema.role_table_grants 
WHERE 1=1 
AND grantee = 'usrqtoapp'
AND table_name='readme_doc'
; 

-- purpose: 
-- list the priveledges per user or for user in a database
-- eof file: src/sql/pgsql/list-table-priviledges.sql
<file_sep>do_chk_install_phantom_js(){
   which phantomjs 2>/dev/null || { 
      sudo apt-get install -y build-essential chrpath libssl-dev libxft-dev
      sudo apt-get install -y libfreetype6 libfreetype6-dev
      sudo apt-get install -y libfontconfig1 libfontconfig1-dev
      export PHANTOM_JS="phantomjs-2.1.1-linux-x86_64"
      wget -O /tmp/$PHANTOM_JS.tar.bz2 https://github.com/Medium/phantomjs/releases/download/v2.1.1/$PHANTOM_JS.tar.bz2
      cd /tmp/
      sudo tar xvjf $PHANTOM_JS.tar.bz2
      sudo mv $PHANTOM_JS /usr/local/share
      sudo ln -sf /usr/local/share/$PHANTOM_JS/bin/phantomjs /usr/local/bin
      phantomjs --version
   }
}
<file_sep>#!/bin/bash

# v0. chk: https://github.com/mojolicious/mojo/wiki/%25ENV
do_mojo_morbo_start(){


  export PATH=$PATH:~/perl5/bin/
  export PERL5LIB=${PERL5LIB:-}:~/perl5/lib/perl5/:$PRODUCT_DIR/src/perl/qto/t/lib:$PRODUCT_DIR/src/perl/qto/public/lib:$PRODUCT_DIR/src/perl/qto/lib

  do_export_json_section_vars $PRODUCT_DIR/cnf/env/$ENV.env.json '.env.app'
  do_mojo_morbo_stop 0

  # to prevent the 'failed: could not create socket: Too many open files at sys' error
  # perl -e 'while(1){open($a{$b++}, "<" ,"/dev/null") or die $b;print " $b"}' to check
  ulimit -n 4096

  sleep "${sleep_interval:=0}"
  export MOJO_LOG_LEVEL='debug'
  export MOJO_MODE='development'
  #export MOJO_MODE='production'
  test -z "${MOJO_MORBO_PORT:-}" && export MOJO_MORBO_PORT='3001'

  export MOJO_LISTEN='http://*:'"$MOJO_MORBO_PORT"
  test -z "${MOJO_MORBO_PORT:-}" && export MOJO_LISTEN='http://*:3001'

  do_log "INFO running: morbo -w $PRODUCT_DIR/src/perl/script/qto
  --listen $MOJO_LISTEN $PRODUCT_DIR/src/perl/qto/script/qto"

  while read -r p ; do
    p=$(echo $p|sed 's/^ *//g')
    test $p -ne $$ && kill -9 $p ;
  done < <(sudo ps -ef | grep -v grep | grep 'mojo-morbo-start'|awk '{print $2}')

  export PERL5LIB=${PERL5LIB:-}:/opt/qto/src/perl/qto/t/lib:/opt/qto/src/perl/qto/public/lib:/opt/qto/src/perl/qto/lib

  bash -c "morbo -w $PRODUCT_DIR/src/perl/qto --listen $MOJO_LISTEN $PRODUCT_DIR/src/perl/qto/script/qto" &

  # might require sudo visudoers
  # usrqtoadmin ALL=(ALL) NOPASSWD: /bin/netstat -tulpn
  do_log "INFO check with netstat, running netstat -tulpn" ; netstat -tulpn | grep qto

}
<file_sep># src/bash/qto/funcs/mojo-morbo-start.test.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doTestMojoMorboStart comments ...
# cat doc/txt/qto/tests/mojo-morbo-start.test.txt
# ---------------------------------------------------------
doTestMojoMorboStart(){

	do_log "INFO START doTestMojoMorboStart"
	
	sleep "$sleep_interval"
	
   # Action !!!
   bash src/bash/qto/qto.sh -a mojo-morbo-start

   # sudo visudoers 
   # ysg ALL=(ALL) NOPASSWD: /bin/netstat -tulpn
   netstat -tulpn | grep qto
	do_log "INFO STOP  doTestMojoMorboStart"
}
# eof func doTestMojoMorboStart


# eof file: src/bash/qto/funcs/mojo-morbo-start.test.sh
<file_sep>#!/usr/bin/env bash

# Some things that run always
export QTO_NO_AUTH=0
export QTO_JWT_AUTH=1

case "$1" in
	start)
   	echo "Starting qto "
		/bin/bash /home/ubuntu/opt/qto/qto.0.8.7.dev.ubuntu@qto.fi/src/bash/qto/qto.sh -a mojo-hypnotoad-start &
		/bin/bash /home/ubuntu/opt/qto/qto.0.8.7.tst.ubuntu@qto.fi/src/bash/qto/qto.sh -a mojo-hypnotoad-start &
		/bin/bash /home/ubuntu/opt/qto/qto.0.8.7.prd.ubuntu@qto.fi/src/bash/qto/qto.sh -a mojo-hypnotoad-start &
    ;;
  stop)
		/bin/bash /home/ubuntu/opt/qto/qto.0.8.7.dev.ubuntu@qto.fi/src/bash/qto/qto.sh -a mojo-hypnotoad-stop &
		/bin/bash /home/ubuntu/opt/qto/qto.0.8.7.tst.ubuntu@qto.fi/src/bash/qto/qto.sh -a mojo-hypnotoad-stop &
		/bin/bash /home/ubuntu/opt/qto/qto.0.8.7.prd.ubuntu@qto.fi/src/bash/qto/qto.sh -a mojo-hypnotoad-stop &
    ;;
  status)
		sudo ps -ef | grep -i qto
    ;;
  *)
    echo "Usage: $0 {start|stop|status}"
    cat << "EOF_ECHO"
      # ---------------------------------------------------
      the sipmliest possible start / stop app script on boot / reboot / shutdown 
      stolen from : https://unix.stackexchange.com/a/57032/37428
      ls -la /etc/init.d/qto-on-boot.sh
      #  -rwxr-x--- 1 root root 947 May 17 15:34 /etc/init.d/qto-on-boot.sh
      # to install : 
      sudo cp -v cnf/bash/etc/init.d/qto-on-boot.sh /etc/init.d/
      sudo update-rc.d qto-on-boot.sh defaults
EOF_ECHO
    exit 1
    ;;
esac

exit 0
<file_sep># src/bash/qto/funcs/mojo-hypnotoad-start.test.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doTestMojoHypnotoadStart comments ...
# ---------------------------------------------------------
doTestMojoHypnotoadStart(){

	do_log "DEBUG START doTestMojoHypnotoadStart"
	
	# cat doc/txt/qto/tests/mojo-hypnotoad-start.test.txt
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!
   bash src/bash/qto/qto.sh -a mojo-hypnotoad-start
   test $exit_code -ne 0 && return

	do_log "DEBUG STOP  doTestMojoHypnotoadStart"
}
# eof func doTestMojoHypnotoadStart


# eof file: src/bash/qto/funcs/mojo-hypnotoad-start.test.sh
<file_sep># # usage: include it in your Makefile
# include <<file-path>>
#
# # function usage example:
#
# uninstall-csitea-web:
# 	$(call uninstall-img,csitea-web,80)


define uninstall-img
	@clear
	-@docker container stop $$(docker ps -aqf "name=${product}-${1}-con") 2> /dev/null
	-@docker container rm $$(docker ps -aqf "name=${product}-${1}-con") 2> /dev/null
endef

<file_sep>// file: scrap-html.js src: https://gist.github.com/magician11/a979906401591440bd6140bd14260578
const CDP = require('chrome-remote-interface');
const chromeLauncher = require('chrome-launcher');

(async function() {
  const launchChrome = () =>
    chromeLauncher.launch({ chromeFlags: ['--disable-gpu', '--headless'] });

  const chrome = await launchChrome();
  const protocol = await CDP({ port: chrome.port });
  const timeout = ms => new Promise(resolve => setTimeout(resolve, ms));

  // See API docs: https://chromedevtools.github.io/devtools-protocol/
  const { Page, Runtime, DOM } = protocol;
  await Promise.all([Page.enable(), Runtime.enable(), DOM.enable()]);

  // ok uri = 'http://qto.fi:8080/qto/view/readme_doc'
  uri = process.argv[2]
  // uri = 'https://qto.fi/qto/view/readme_doc'
  // uri = 'http://host-name:8082/tst_kone/list/requirements_doc?&pg-size=100&pick=name,formats'
  console.log ( uri )
  Page.navigate({ url: uri });

  // wait until the page says it's loaded...
  Page.loadEventFired(async () => {
    try {
      await timeout(4000); // give the JS some time to load

      // get the page source
      const rootNode = await DOM.getDocument({ depth: -1 });
      const pageSource = await DOM.getOuterHTML({
        nodeId: rootNode.root.nodeId
      });
      protocol.close();
      chrome.kill();
      console.log ( pageSource.outerHTML)
    } catch (err) {
      console.log(err);
    }
  });
})();
//eof file: scrap-html.js
<file_sep>-- DROP TABLE IF EXISTS questions ; 

SELECT 'create the "questions" table'
; 
   CREATE TABLE questions (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , prio           integer NOT NULL DEFAULT 1
    , questions_status_guid UUID NOT NULL DEFAULT 'cb989a14-d0b8-46e4-b2cc-5e2a974b5d29'
    , category       varchar (20) NOT NULL DEFAULT 'unknown'
    , name           varchar (100) NOT NULL DEFAULT 'name ...'
    , description    varchar (4000) NOT NULL DEFAULT 'description ... '
    , owner          varchar (20) NULL 
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_questions_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.questions'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 


--The trigger:
CREATE TRIGGER trg_set_update_time_on_questions BEFORE UPDATE ON questions FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'questions'::regclass;
<file_sep>#!/bin/bash

do_create_app_user(){
   set +e

   test -z ${PROJ_CONF_FILE:-} && export PROJ_CONF_FILE="$PRODUCT_DIR/cnf/env/$ENV.env.json"
   source $PRODUCT_DIR/lib/bash/funcs/export-json-section-vars.sh
   do_export_json_section_vars $PROJ_CONF_FILE '.env.db'
   pgsql_scripts_dir="$PRODUCT_DIR/src/sql/pgsql/qto"
   tmp_log_file="/tmp/$$.create-app-user.log"

   # 01 create / modify the app user
   sql_script="$pgsql_scripts_dir/01.create-app-db-user.pgsql"
   #sql_script="$pgsql_scripts_dir/02.alter-qto-app-user.pgsql"
   PGPASSWORD="${postgres_sys_usr_admin_pw:-}" psql -q -t -X -w -U "${postgres_sys_usr_admin:-}" \
      -h $postgres_rdbms_host -p $postgres_rdbms_port -v ON_ERROR_STOP=1 \
      -v postgres_app_usr="${postgres_app_usr:-}" \
      -v postgres_app_usr_pw="${postgres_app_usr_pw:-}" \
      -v postgres_app_db="${postgres_app_db:-}" \
      -f "$sql_script" "${postgres_app_db:-}" > "$tmp_log_file" 2>&1
   ret=$?

   cat "$tmp_log_file" ; cat "$tmp_log_file" >> $log_file # show it and save it
   test $ret -ne 0 && {
      sleep 3 ; do_exit 1 "pid: $$ psql ret $ret - failed to run sql_script: $sql_script !!!";
   }
   PGPASSWORD="${postgres_sys_usr_admin_pw:-}" psql -q -t -X -w -U "${postgres_sys_usr_admin:-}" \
      -h $postgres_rdbms_host -p $postgres_rdbms_port -d postgres -v ON_ERROR_STOP=1 -c \
      "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO $postgres_app_usr;
	  GRANT CONNECT ON DATABASE $postgres_app_db TO $postgres_app_usr;
	  GRANT SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA public TO $postgres_app_usr;
      GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO $postgres_app_usr;"

}
<file_sep># src/bash/qto/funcs/db-to-xls.test.sh

# v0.6.9
# ---------------------------------------------------------
# the wrapper func around the db to xls conversion
# ---------------------------------------------------------
doTestDbToXls(){

	do_log "DEBUG START doTestDbToXls"
	
	cat doc/txt/qto/tests/db-to-xls.test.txt
	
	sleep "$sleep_interval"
	# Action !!!
   src/bash/qto/qto.sh -a db-to-xls
   
   exit_code=$?
	do_log "INFO txt-to-db.test-1 exit_code: $exit_code "
   sleep "$sleep_interval"
   test $exit_code -ne 0 && return

	do_log "DEBUG STOP  doTestDbToXls"
}


# eof file: src/bash/qto/funcs/db-to-xls.test.sh
<file_sep>#!/usr/bin/env bash

set -eu -o pipefail # fail on error , debug all lines

set -x 

# run as root
#[ "$USER" = "root" ] || exec sudo "$0" "$@"

doWgetMinioBins(){
	test -f /root/minio || wget -O /root/minio https://dl.minio.io/server/minio/release/linux-amd64/minio
	test -f /root/mc || wget -O /root/mc https://dl.minio.io/client/mc/release/linux-amd64/mc
   chmod 755 /root/minio
   chmod 755 /root/mc
   ls -la /root/minio
   ls -la /root/mc
}

echo "installing the must-have pre-requisites"
while read -r p ; do
   if [ "" == "`which $p`" ]; then 
      doWgetMinioBins
   fi
done < <(cat << "EOF01"
	grep
	lsof
	pgrep
	openssl
	minio
EOF01
)


export minio_data_dir='/tmp/minio-data'

# set credentials for the server
export MINIO_ACCESS_KEY='N<KEY>'
export MINIO_SECRET_KEY='<KEY>'
export MINIO_REGION="local_region"

echo MINIO_ACCESS_KEY: $MINIO_ACCESS_KEY
echo MINIO_SECRET_KEY: $MINIO_SECRET_KEY
echo MINIO_REGION: $MINIO_REGION

mkdir -p ~/etc/ ; cat << "EOF02" > ~/etc/openssl.conf
[req]
distinguished_name = req_localhost
x509_extensions = v3_req
prompt = no

[req_localhost]
C = FI
ST = UU
L = Helsinki
O = Corp
OU = org
CN = localhost

[v3_req]
subjectAltName = @alt_names

[alt_names]
IP.1 = 127.0.0.1
EOF02

test $(pgrep minio|wc -l) -gt 0 && kill -9 $(pgrep minio) # kill if running

mkdir -p ~/.minio/certs/ ;
test -f ~/.minio/certs/private.key && rm -v ~/.minio/certs/private.key
openssl genrsa -out ~/.minio/certs/private.key 2048

test -f ~/.minio/certs/public.crt && rm -v ~/.minio/certs/public.crt
openssl req -x509 -nodes -days 730 -newkey rsa:2048 -keyout ~/.minio/certs/private.key \
   -out ~/.minio/certs/public.crt -config ~/etc/openssl.conf

/root/minio server "$minio_data_dir" 2>&1 &  # and start the server

export CONFIG_S3_BUCKET_NAME='org-bucket'
export CONFIG_S3_ENDPOINT='https://127.0.0.1:9000'
export CONFIG_S3_ACCESS_KEY=$MINIO_ACCESS_KEY
export CONFIG_S3_SECRET_KEY=$MINIO_SECRET_KEY
export CONFIG_S3_REGION=$MINIO_REGION
export CONFIG_S3_DISABLE_SSL_CHECK='true'

echo CONFIG_S3_BUCKET_NAME: $CONFIG_S3_BUCKET_NAME
echo CONFIG_S3_ENDPOINT: $CONFIG_S3_ENDPOINT
echo CONFIG_S3_ACCESS_KEY: $CONFIG_S3_ACCESS_KEY
echo CONFIG_S3_SECRET_KEY: $CONFIG_S3_SECRET_KEY
echo CONFIG_S3_REGION: $CONFIG_S3_REGION
echo CONFIG_S3_DISABLE_SSL_CHECK: $CONFIG_S3_DISABLE_SSL_CHECK

export MINIO_HOST_ALIAS='minio_host_alias'
# register the host alias
/root/mc config host add $MINIO_HOST_ALIAS "$CONFIG_S3_ENDPOINT" "$CONFIG_S3_ACCESS_KEY" \
	"$CONFIG_S3_SECRET_KEY" --api S3v4

# create the bucket
/root/mc --insecure mb --region "$MINIO_REGION" "$MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME"

# configure the policies on the bucket
/root/mc --insecure policy upload "$MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME"
/root/mc --insecure policy download "$MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME"
/root/mc --insecure policy public "$MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME"
/root/mc --insecure policy list "$MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME"

# ls the buckets in the host alias
/root/mc --insecure ls $MINIO_HOST_ALIAS

# copy the resource file ysg:todo add livy.conf to this script
/root/mc --insecure cp main/src/test/resources/livy/livy.conf $MINIO_HOST_ALIAS/$CONFIG_S3_BUCKET_NAME

# and verify it is there
/root/mc --insecure stat $MINIO_HOST_ALIAS/org-bucket/livy.conf

mkdir -p ~/.aws/s3cmd/ ; cat << EOF03 > ~/.aws/s3cmd/minio_local.s3cfg
  host_base = 127.0.0.1:9000
  host_bucket = org-bucket
  bucket_location = local_region
  use_https = True
  AWS_ACCESS_KEY_ID =  <KEY>
  AWS_SECRET_ACCESS_KEY = <KEY>
  signature_v2 = False
EOF03

# run those manually
# test $(which s3cmd|wc -l) -gt 0 && \
#   s3cmd ls --no-check-certificate -r -c ~/.aws/s3cmd/minio_local.s3cfg
# "s3://$CONFIG_S3_BUCKET_NAME"

#AWS_ACCESS_KEY="$MINIO_ACCESS_KEY"
#AWS_SECRET_KEY="$MINIO_SECRET_KEY"
#aws s3 ls --region "$MINIO_REGION" --endpoint-url "$CONFIG_S3_ENDPOINT" --no-verify-ssl true


<file_sep>
do_chk_install_python_modules(){
   cd $PRODUCT_DIR

   
   sudo apt-get install -y python3-venv
   pip3 install virtualenv ; python3 -m venv ./venv
   pip install --ignore-installed six
   pip3 install -r $PRODUCT_DIR/cnf/bin/python/requirements.txt

	sleep 1 ; do_flush_screen 
}
<file_sep># src/bash/qto/funcs/morph-dir.help.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doHelpMorphDir comments ...
# ---------------------------------------------------------
doHelpMorphDir(){

	do_log "DEBUG START doHelpMorphDir"
	
	cat doc/txt/qto/helps/morph-dir.help.txt
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!

	do_log "DEBUG STOP  doHelpMorphDir"
}
# eof func doHelpMorphDir


# eof file: src/bash/qto/funcs/morph-dir.help.sh
<file_sep>#!/bin/bash


# ---------------------------------------------------------
# stop any running morbo instances listenning on the wanted port
# cat doc/txt/qto/funcs/mojo-morbo-stop.func.txt
# ---------------------------------------------------------
do_mojo_morbo_stop(){

  export PATH=$PATH:~/perl5/bin/
  export PERL5LIB=${PERL5LIB:-}:~/perl5/lib/perl5/:$PRODUCT_DIR/src/perl/qto/t/lib:$PRODUCT_DIR/src/perl/qto/public/lib:$PRODUCT_DIR/src/perl/qto/lib

  do_export_json_section_vars $PRODUCT_DIR/cnf/env/$ENV.env.json '.env.app'
  test -z "${MOJO_MORBO_PORT:-}" && export MOJO_MORBO_PORT=3001

   while read -r listening_on_port_pid; do
      while read -r pid; do
         test $listening_on_port_pid -eq $pid && sudo kill -9 $pid
      done < <(sudo ps -ef| grep -i qto| grep -v grep|awk '{print $2}')
   done < <(lsof -i:${MOJO_MORBO_PORT:-} -t)

   echo start running ps -ef \| grep qto
   ps -ef | grep -i qto
   echo stop  running ps -ef \| grep qto
   echo -e "\n"
   export exit_code=0
}
<file_sep># Makefile
# usage: run the "make" command in the root, than make <<cmd>>...
include $(wildcard lib/make/*.mk)
include $(wildcard src/make/*.mk)

# get those from the env when in azure, otherwise get locally
BUILD_NUMBER := $(if $(BUILD_NUMBER),$(BUILD_NUMBER),"0")
COMMIT_SHA := $(if $(COMMIT_SHA),$(COMMIT_SHA),$$(git rev-parse --short HEAD))
COMMIT_MESSAGE := $(if $(COMMIT_MESSAGE),$(COMMIT_MESSAGE),$$(git log -1  --pretty='%s'))

SHELL = bash
PRODUCT := $(shell basename $$PWD)


.PHONY: install ##
install: install-devops


# eof Makefile
<file_sep># src/bash/qto/funcs/run-unit-tests.test.sh

# v0.6.9
doTestRunUnitTests(){

	do_log "DEBUG START doTestRununitTests"
	
	sleep "$sleep_interval"
   # Action !!!	
	bash src/bash/qto/qto.sh -a run-unit-tests
	
	do_log "DEBUG STOP  doTestRununitTests"
}


# eof file: src/bash/qto/funcs/run-unit-tests.test.sh
<file_sep># src/bash/qto/funcs/log-test-run-entry.test.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doTestLogTestRunEntry comments ...
# ---------------------------------------------------------
doTestLogTestRunEntry(){

	do_log "DEBUG START doTestLogTestRunEntry"
	
	cat doc/txt/qto/tests/log-test-run-entry.test.txt
	
	sleep "$sleep_interval"

   action_n='some-action-name'

   # should initialize the test run report
   do_logTestRunEntry 'INIT'

   do_logTestRunEntry 'START'
   
   # should register the test line start time
   do_logTestRunEntry 'INFO' ' '$(date "+%H:%M:%S")
   
   # should register the test line stop time
   do_logTestRunEntry 'INFO' ' '$(date "+%H:%M:%S")
   
   # should addd the name of the action to test 
   do_logTestRunEntry 'INFO' ' '"$action_n"
   
   # should close the test run line
   do_logTestRunEntry 'STOP' 0
   
   # should close and print the test run report 
   do_logTestRunEntry 'STATUS'

	do_log "DEBUG STOP  doTestLogTestRunEntry"
}
# eof func doTestLogTestRunEntry


# eof file: src/bash/qto/funcs/log-test-run-entry.test.sh
<file_sep># src/bash/qto/funcs/print-usage.test.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doTestPrintUsage comments ...
# ---------------------------------------------------------
doTestPrintUsage(){

	do_log "DEBUG START doTestPrintUsage"
	
	cat doc/txt/qto/tests/print-usage.test.txt
	sleep 2
	# add your action implementation code here ... 

	do_log "DEBUG STOP  doTestPrintUsage"
}
# eof func doTestPrintUsage


# eof file: src/bash/qto/funcs/print-usage.test.sh
<file_sep>FROM python:3.8.3-alpine


ARG ENV
ARG UID
ARG GID
ARG PRODUCT
ARG TGT_ORG

ENV PRODUCT=$PRODUCT
ENV TGT_ORG=$TGT_ORG
ENV APPUSR=appusr
ENV APPGRP=appgrp
ENV PS1="`date \"+%F %T\"` \u@\h  \w \n\n  "
ENV PRODUCT_DIR="/opt/$PRODUCT"
ENV EDITOR="vim"
ENV ENV=$ENV
ENV TERM="xterm-256color"

VOLUME $PRODUCT_DIR

# START ::: install bins
RUN apk update && \
    apk upgrade && \
    apk add --no-cache \
    bash binutils vim perl jq wget \
    curl zip unzip busybox-extras \
    su-exec sudo shadow net-tools \
    build-base gcc openssl-dev git \
    libmagic ttf-freefont make jq \
    python3-dev jpeg-dev zlib-dev

# STOP ::: install bins

RUN pip install wheel

# START ::: Enable host to container edit of proj code on ubuntu and mac.
RUN test -z $(getent group $GID | cut -d: -f1) || \
        groupmod -g $((GID+1000)) $(getent group $GID | cut -d: -f1)

RUN set -x ; addgroup -g "$GID" -S "$APPGRP" && \
  adduser \
  --disabled-password \
  -g "$GID" \
  -D \
  -s "/bin/bash" \
  -h "/home/$APPUSR" \
  -u "$UID" \
  -G "$APPGRP" "$APPUSR" && exit 0 ; exit 1

RUN echo "$APPUSR ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
RUN echo "export PS1=\"$PS1\"" >> /home/$APPUSR/.bashrc

USER $APPUSR
ENV USER=$APPUSR
ENV GROUP=$APPGRP

# STOP  ::: enable host to container edit of proj code on ubuntu and mac.

ADD --chown=$APPUSR:$APPGRP "." "/home/$APPUSR$PRODUCT_DIR"

# stop ::: adding OS APPUSER and APPGROUP


USER $APPUSR
WORKDIR $PRODUCT_DIR


# install awscli
RUN pip3 install --upgrade pip \
      && pip3 install --no-cache-dir awscli \
      && sudo rm -rf /var/cache/apk/* \
      && sudo apk add aws-cli \
      && aws --version

# add project folder
ADD --chown=$APPUSR:$APPGRP "." /home/$APPUSR$PRODUCT_DIR

# but use the proj root dir to write the code on and interact
WORKDIR $PRODUCT_DIR

CMD exec /bin/bash -c "/home/$APPUSR/$PRODUCT_DIR/src/bash/run/docker-init-devops.sh"
<file_sep>#!/usr/bin/env bash

if [ \( "$1" = '--usage' \) -o \( "$1" = '' \) -o \( "$1" = '--help' \) ]
then
   source $PRODUCT_DIR/lib/bash/funcs/flush-screen.sh
   do_flush_screen
   
   url='https://qto.fi:442/qto/select/monthly_issues_202005?oa=prio&with=app_users_guid-eq-02d16010-20af-4b0d-be86-cdf116a7d8c7'
   tgt_table='monthly_issues_202005'
   cat << EOF_PRINT_USAGE

   $0
   Purpose: 
   --------------------------------------------------------
   upsert qto http json data to postgres db

   USAGE EXAMPLE:
   --------------------------------------------------------
  
   url='https://qto.fi:442/qto/select/monthly_issues_202005'
   tgt_table=monthly_issues_202005
   bash $0 \$url \$tgt_table


EOF_PRINT_USAGE
   exit 1
fi

#https://qto.fi:442/qto/select/monthly_issues_202005?oa=prio' | jq
url=$(echo $1|perl -ne 's|\/list|/select|g;print') # copy list from browser but use select for data
echo $url
#curl -s -b ~/.qto/cookies.txt "$url"
#sleep 3
export json_str=$(curl -s -b ~/.qto/cookies.txt "$url" | jq -r '.dat')
export itm=$2

curl -s -b ~/.qto/cookies.txt "$url" | jq -r '.dat'

echo $json_str
IFS='' read -r -d '' perl_code <<"EOF_PERL_CODE"
      use strict; use warnings; binmode STDOUT, ":utf8"; use utf8; 
      use JSON; use Data::Printer;use Mojo::Template; use feature ':5.12';
      my $json_var = decode_json($ENV{'json_str'}) ; 
      my $itm = $ENV{'itm'};
      #p $json_var ;
      my $col_lst = '(';
      my $val_lst = '(';
      my $upd_lst = '';
      my $sql_lst = '';
      foreach my $row (@$json_var){
         #p $row ; 
         foreach my $key (keys %$row) {
            my $cll = $row->{$key} ;
            $cll = '' if ($cll eq 'null' or !defined($cll));
            $cll =~ s/'/''/g;
            $cll =~ s/\\n/\\\n/g;
            $col_lst .= $key . ', ' ;
            $upd_lst .= $key . ' = ' . "'" . $cll . "'" . ', ';
            $val_lst .= "'" . $cll . "'" . ', ' ;
         }
         $col_lst = substr($col_lst, 0,-2); $col_lst .= ')'; 
         $upd_lst = substr($upd_lst, 0,-2); 
         $val_lst = substr($val_lst, 0,-2); $val_lst .= ')';
         $sql_lst .= "INSERT INTO $itm $col_lst VALUES $val_lst ON CONFLICT (id) DO UPDATE SET $upd_lst; ";
         $col_lst = '(';
         $val_lst = '(';
         $upd_lst = '';
      }
      #p $col_lst ; 
      #p $val_lst ; 
      my $mt = Mojo::Template->new;
      say $mt->render(<<'EOF', $sql_lst, $col_lst,$val_lst, $upd_lst, $ENV{'col_lst_excluded'});
      % my ($sql_lst, $col_lst, $val_lst, $upd_lst, $col_lst_excluded) = @_;
      <%= $sql_lst %>
EOF


EOF_PERL_CODE

#echo $perl_code
#echo eof perl_code
sql_code=$(perl -e "$perl_code")
echo $sql_code
PGPASSWORD=${postgres_sys_usr_admin_pw:-} psql -v -t -X -w -U ${postgres_sys_usr_admin:-} \
            --port $postgres_rdbms_port --host $postgres_rdbms_host -t -d ${postgres_app_db:-} \
            -c "$sql_code"
<file_sep># vi: set ft=ruby :

ENV['VAGRANT_DEFAULT_PROVIDER'] = 'virtualbox'



Vagrant.configure("2") do |config|

  # dev, tst and prd are separate vagrants
  config.vm.define "%ENV%-qto-vagrant" do |config|
  config.vm.hostname = "%ENV%-qto-vagrant"

  # use the latest ubuntu as a base
  config.vm.box = "generic/ubuntu1804"

  # disable updates to the configured box
  config.vm.box_check_update = false

  # dev, tst and prd could run on the same host if ip is changed ...
  # your mileage might vary ...
  config.vm.network "private_network", ip: "192.168.10.50"

  # open the the morbo dev web server ports for dev, tst and prd
  config.vm.network "forwarded_port", guest: %dev_mojo_morbo_port%, host: %dev_mojo_morbo_port%
  config.vm.network "forwarded_port", guest: %tst_mojo_morbo_port%, host: %tst_mojo_morbo_port%
  config.vm.network "forwarded_port", guest: %prd_mojo_morbo_port%, host: %prd_mojo_morbo_port%

  # open the the hypno prd web server ports for dev, tst and prd
  config.vm.network "forwarded_port", guest: %dev_mojo_hypnotoad_port%, host: %dev_mojo_hypnotoad_port%
  config.vm.network "forwarded_port", guest: %tst_mojo_hypnotoad_port%, host: %tst_mojo_hypnotoad_port%
  config.vm.network "forwarded_port", guest: %prd_mojo_hypnotoad_port%, host: %prd_mojo_hypnotoad_port%
  

  # the project dir will be visible from the vagrant @/home/vagrant/opt/qto
  config.vm.synced_folder "%product_base_dir%", "/home/vagrant/opt/"
  $script = <<-SCRIPT
         # run the bootstrap script and IMPORTANT !!! reload the bash shell
         bash /home/vagrant/opt/qto/src/bash/deployer/run.sh

         bash

         # go to the product instance dir
         cd /home/vagrant/opt
         source $(find . -name '.env') && cd qto/qto.$VERSION.$ENV.$USER

         # ensure application layer consistency, run db ddl's and load data from s3
         bash ./src/bash/qto/qto.sh -a check-perl-syntax -a scramble-confs 
         bash ./src/bash/qto/qto.sh -a provision-db-admin -a run-qto-db-ddl 
         bash ./src/bash/qto/qto.sh -a load-db-data-from-s3 -a fix-db-permissions

         # start the web server
         ./src/bash/qto/qto.sh -a mojo-hypnotoad-start
  SCRIPT

  # and run the script
  config.vm.provision "shell", inline: $script, privileged: false
  end
end
<file_sep>#  MAINTENANCE GUIDE
* [1. GUIDING PRINCIPLE'S](#1-guiding-principle's)
  * [1.1. PERSONAL RESPONSIBILITY](#11-personal-responsibility)
* [2. RUN-TIMES STATES CHANGES](#2-run-times-states-changes)
  * [2.1. CHECK THE STATUS OF THE POSTGRESQL](#21-check-the-status-of-the-postgresql)
  * [2.2. STOP THE POSTGRESQL](#22-stop-the-postgresql)
  * [2.3. START THE POSTGRESQL](#23-start-the-postgresql)
  * [2.4. CHECK THE PORT ON WHICH IT IS LISTENING ](#24-check-the-port-on-which-it-is-listening-)
  * [2.5. CHECK THE POSTGRES STATUS](#25-check-the-postgres-status)
  * [2.6. APPLICATION LAYER RUN-STATE MANAGEMENT](#26-application-layer-run-state-management)
    * [2.6.1. Start the application layer](#261-start-the-application-layer)
    * [2.6.2. Stop the application layer](#262-stop-the-application-layer)
    * [2.6.3. Restart OS network service](#263-restart-os-network-service)
  * [2.7. SECURITY RELATED OPERATIONS](#27-security-related-operations)
    * [2.7.1. Add, modify and delete new users to the application](#271-add-modify-and-delete-new-users-to-the-application)
    * [2.7.2. Regular users visibility](#272-regular-users-visibility)
* [3. MODIFY THE UI FROM THE DB DATA](#3-modify-the-ui-from-the-db-data)
  * [3.1. HIDE A COLUMN FROM THE LIST PAGE GLOBALLY FOR ALL USERS](#31-hide-a-column-from-the-list-page-globally-for-all-users)
  * [3.2. SET THE WIDTH OF AN ITEM COLUMN IN THE LISTING PAGE](#32-set-the-width-of-an-item-column-in-the-listing-page)
  * [3.3. ADD NEW FK DATA FOR THE DROP DOWNS IN THE LIST PAGE](#33-add-new-fk-data-for-the-drop-downs-in-the-list-page)
* [4. BACKUP AND RESTORE PROJECTS DATA](#4-backup-and-restore-projects-data)
  * [4.1. BACKUP A DATABASE](#41-backup-a-database)
  * [4.2. LOAD DATABASE CONNECTIVITY CONFIGURATION SECURELY](#42-load-database-connectivity-configuration-securely)
  * [4.3. BACKUP A DATABASE TABLE](#43-backup-a-database-table)
  * [4.4. BACKUP ALL THE DATA FROM A REMOTE HOST TO LOCALHOST](#44-backup-all-the-data-from-a-remote-host-to-localhost)
  * [4.5. REMOVING OLD BACKUPS](#45-removing-old-backups)
  * [4.6. BACKUP ONLY THE DATABASE TABLES INSERT DATA](#46-backup-only-the-database-tables-insert-data)
  * [4.7. RESTORE A DATABASE](#47-restore-a-database)
    * [4.7.1. Restore a database from full database backup](#471-restore-a-database-from-full-database-backup)
    * [4.7.2. Restore a database from db inserts file](#472-restore-a-database-from-db-inserts-file)
  * [4.8. RESTORE A DATABASE TABLE](#48-restore-a-database-table)
  * [4.9. CREATE A FULL BACKUP OF AN INSTANCE SITE FROM A SATELLITE HOST](#49-create-a-full-backup-of-an-instance-site-from-a-satellite-host)
  * [4.10. BACKUP MULTIPLE EC2 SERVERS FROM THE SATELLITE HOST](#410-backup-multiple-ec2-servers-from-the-satellite-host)
* [5. SHELL ACTIONS](#5-shell-actions)
  * [5.1. BACKUP DIRECTORIES WITH RSYNC](#51-backup-directories-with-rsync)
  * [5.2. LOAD XLS SHEET TO DB A DOC TABLE](#52-load-xls-sheet-to-db-a-doc-table)
  * [5.3. RUN INCREASE-DATE ACTION](#53-run-increase-date-action)
* [6. NGINX CONFIGURATION](#6-nginx-configuration)
  * [6.1. THE NGINX PROVISIONING](#61-the-nginx-provisioning)
  * [6.2. APPLYING NEW NGINX CONFIGURATION](#62-applying-new-nginx-configuration)
  * [6.3. OPENING A MONTHLY PERIOD](#63-opening-a-monthly-period)
  * [6.4. CLOSING A MONTHLY PERIOD](#64-closing-a-monthly-period)
* [7. BULK ISSUES MANAGEMENT](#7-bulk-issues-management)
  * [7.1. DIRS NAMING CONVENTIONS](#71-dirs-naming-conventions)
  * [7.2. PUBLISH THE RELEASE ISSUES](#72-publish-the-release-issues)
* [8. NAMING CONVENTIONS](#8-naming-conventions)
  * [8.1. NAMING CONVENTIONS FOR DIRECTORIES](#81-naming-conventions-for-directories)
  * [8.2. ROOT DIRS NAMING CONVENTIONS](#82-root-dirs-naming-conventions)
* [9. SOURCE CODE MANAGEMENT](#9-source-code-management)
  * [9.1. CONFIGURE AND USE GIT ALWAYS BY USING SSH IDENTITIES](#91-configure-and-use-git-always-by-using-ssh-identities)
  * [9.2. AIM FOR E2E TRACEABILITY](#92-aim-for-e2e-traceability)
  * [9.3. RESTART THE APPLICATION LAYER](#93-restart-the-application-layer)
* [10. KNOWN ISSUES AND WORKAROUNDS](#10-known-issues-and-workarounds)
  * [10.1. MORBO IS STUCK](#101-morbo-is-stuck)
    * [10.1.1. Symptoms](#1011-symptoms)
    * [10.1.2. Probable root cause](#1012-probable-root-cause)
    * [10.1.3. Known solution and workaround](#1013-known-solution-and-workaround)
  * [10.2. NGINX PRESENTS HTTP 502](#102-nginx-presents-http-502)
* [11. LINUX SYSTEM ADMINISTRATION](#11-linux-system-administration)
  * [11.1. CHECK OPEN FILE DESCRIPTORS](#111-check-open-file-descriptors)
* [12. SECURITY](#12-security)
  * [12.1. CHECK THE LOGIN EVENTS ON A QTO INSTANCE](#121-check-the-login-events-on-a-qto-instance)




    

## 1. GUIDING PRINCIPLE'S
This section might seem too philosophical for a start, yet all the development in the qto has ATTEMPTED to follow the principles described below. If you skip this section now you might later on wander many times why something works and it is implemented as it is ... and not "the right way". 
Of course you are free to not follow these principles, the less you follow them the smaller the possibility to pull features from your instance(s) - you could even use the existing functionality to create a totally different fork with different name and start developing your own toll with name X - the authors give you the means to do that with this tool ... , but if you want to use and contribute to THIS tool than you better help defined those leading principles and follow them. 

    

### 1.1. Personal responsibility
Any given instance of the qto should have ONE and only ONE person which is responsible at the end for the functioning of THE instance - so think carefully before attempting to take ownership of an instance. The author(s) of the code are not responsible for the operation, bugs or whatever happens to a new instance. As a responsible owner of an instance you could create, share and assign issues to the authors of the source code, yet there is no Service Level Agreement, not even promise to help. 

    

## 2. RUN-TIMES STATES CHANGES


    

### 2.1. Check the status of the postgreSql
To check the status of the postgreSql issue:

    sudo /etc/init.d/postgresql status

### 2.2. Stop the postgreSql
To stop the postgreSql issues:

    sudo /etc/init.d/postgresql stop

### 2.3. Start the postgreSql
To start the postgreSql issues:

    sudo /etc/init.d/postgresql start

### 2.4. Check the port on which it is listening 
To check the port on which it is listening issue:





    sudo netstat -tulntp | grep -i postgres
    # tcp        0      0 127.0.0.1:5432          0.0.0.0:*               LISTEN      8095/postgres

### 2.5. Check the postgres status
Check the postgres status.
Check the port to which the Postgres is running with this command:



    sudo /etc/init.d/postgresql status
    
    # restart if needed
    sudo /etc/init.d/postgresql restart
    
    # check on which ports it is runnning
    sudo netstat -plunt |grep postgres

### 2.6. Application Layer run-state management
Remember to cd to the product instance dir you are going to work on for example: 
cd /vagrant/opt/csitea/qto/qto.0.6.5.dev.ysg
All the examples below are assuming you've done that in advance. 

    

#### 2.6.1. Start the application layer
To start the application layer in development mode use the morbo command ( debug output will be shown ) , to start it in production mode use the hypnotoad pattern 

    # start hypnotoad ( does the stop as well )
    bash src/bash/qto/qto.sh -a mojo-hypnotoad-start
    
    # start morbo
    bash src/bash/qto/qto.sh -a mojo-morbo-start

#### 2.6.2. Stop the application layer
To stop the application layer in development mode use the morbo command ( debug output will be shown ) , to start it in production mode use the hypnotoad pattern. Note that the morbo command does not stop any running morbo on OTHER product instance dir, but the hypnotoad does stop all - aka hypntotoad as the binary of production must be running only on 1 and only one instance on a host.

    # only stop hypno
    bash src/bash/qto/qto.sh -a mojo-hypnotoad-stop
    
    # only stop morbo
    bash src/bash/qto/qto.sh -a mojo-morbo-stop

#### 2.6.3. Restart OS network service
Sometimes you might just need to restart the whole network service on Ubuntu:

    sudo /etc/init.d/networking restart
    # or 
    bash src/bash/qto/qto.sh -a restart-network # this one restarts the postgres as well 

### 2.7. Security related operations
There are 2 security modes of operations in qto:
 - none authenticative one ( no login , all can be changed by anyone )
 - native authentication mode - the user credentials are stored per db

    

#### 2.7.1. Add, modify and delete new users to the application
You as the owner of the instance you are running must be aware that the requests to register to the instances you are operating will come via e-mail. 
Simply add, update and delete users in the users table and sent the password with prompt to edit it to the new user.

    

#### 2.7.2. Regular users visibility
Use the following http password generator:
http://www.htaccesstools.com/htpasswd-generator/

    

## 3. MODIFY THE UI FROM THE DB DATA
Qto is data driven application - this means that certain element of the UI could be changed by changing certain data in the application

    

### 3.1. Hide a column from the list page globally for all users
Open the app_item_attributes table, type the item's table_name and the column_name, type 1 in the ski_in_list column.

    

### 3.2. Set the width of an item column in the listing page
Open the app_item_attributes table, type the item's table_name and the column_name, type a desired number for the width ( 60 might be good default to start experimenting with)

    

### 3.3. Add new FK data for the drop downs in the list page
Locate the primary key table which holds the FK data from the listed item you want to modify the drop down data to. Add new rows in this table, the new rows will be visible straight away in the UI, and if not - the users should logout and login to clear their localStorage. Note, that some browsers do not show properly most the dropdown select's scrollbars if more than 12 items are contained in the select - aka the users should scroll till the end of the dropdown to see the full list of items ...

    

## 4. BACKUP AND RESTORE PROJECTS DATA
You could easily add those commands to your crontab for scheduled execution - remember to add the absolution patch of the qto.sh entry script.
Anything you perform as shell action in qto could be applied not only to the current product instance dir, but also any other project instance dir, which has a directory structure, which is compatible with the current qto product.

    

### 4.1. Backup a database
You backup a database ( all the objects, app_roles and data ) with the following one-liner. 

    # obs you must have the shell vars preloaded !!! Note dev, tst or prd instances !
    # clear; doParseCnfEnvVars cnf/qto.prd.host-name.cnf
    bash src/bash/qto/qto.sh -a backup-postgres-db

### 4.2. Load database connectivity configuration securely
Qto provides you with the means and tools to work on tens of databases, yet one at the time. Thus once you open a shell to run the tools you must have the connectivity to the database you want to work on. 

    source lib/bash/funcs/export-json-section-vars.sh
    # optionally use a different project than the current product instance dir
    export PROJ_INSTANCE_DIR=/hos/opt/kone/kone-qto
    
    # optionally use a differnt configuration file for this proj instance dir
    export PROJ_CONF_FILE=/hos/opt/org/org-qto/cnf/env/tst.env.json
    
    # load the env vars from this project
    doExportJsonSectionVars $PROJ_CONF_FILE '.env.db'
    
    # set the psql with the correct credentials valid ONLY for this proj
    alias psql="PGPASSWORD=${postgres_sys_usr_admin_pw:-} psql -v -t -X -w -U ${postgres_sys_usr_admin:-} --port $postgres_rdbms_port --host $postgres_rdbms_host"
    
    # now you can run any psql 
    psql -d my_db -c "\dl"

### 4.3. Backup a database table
You backup a database table with the following one-liner. Noe 

    # obs you have to have the shell vars preloaded !!!
    # clear; doParseCnfEnvVars <<path-to-cnf-file>>
    bash src/bash/qto/qto.sh -a backup-postgres-table -t my_table

### 4.4. Backup all the data from a remote host to localhost
Storage is cheap - human time is expensive - this the principle according to which the backup ideology of qto has been build upon ... and for now it is working ...
Because of the naming convention of the product instance directories you will never overwrite a local directory with the remote directories and always keep track of what where is.
So provided that you have configured an ssh key to access you remote server you would issue the following one-liner which will replicate the remote structure on the remote host on the localhost ...

    export ssh_server=qto.fi
    export ssh_user=ubuntu
    export src_dir=/home/ubuntu/opt/qto/
    export tgt_dir=/hos/opt/
    rsync -e "ssh -l USERID -i ~/.ssh/id_rsa.aws-ec2.qto.prd" -av -r --partial --progress --human-readable \
    	--stats $ssh_user@$ssh_server:$src_dir $tgt_dir

### 4.5. Removing old backups
Because of the naming convention removing backups requires as little effort as: 

    # check what where is in the localhost
    find /hos/opt/qto -type d -maxdepth 1 -mindepth 1 | sort -nr | less
    
    # remove ALL dirs an files belonging to v0.8.1, as we are now in v0.8.4
    rm -r /hos/opt/qto/qto.0.8.1*
    
    # and verify
    find /hos/opt/qto -type d -maxdepth 1 -mindepth 1 | sort -nr | less
    # remove ALL dirs an files belonging to v0.8.2, as we are now in v0.8.4
    rm -r /hos/opt/qto/qto.0.8.2*
    
    # and verify
    find /hos/opt/qto -type d -maxdepth 1 -mindepth 1 | sort -nr | less 
    

### 4.6. Backup only the database tables insert data


    # obs you have to have the shell vars preloaded !!!
    bash src/bash/qto/qto.sh -a backup-postgres-db-inserts

### 4.7. Restore a database
You restore a database by first running the pgsql scripts of the project database and than restoring the insert data 

    psql -d $postgres_app_db < \
    dat/mix/sql/pgsql/dbdumps/dev_qto/dev_qto.20180813_202202.insrt.dmp.sql

#### 4.7.1. Restore a database from full database backup
You restore a database by basically creating first the empty database and applying the dump file to that database. Practice this several times in dev before going to tst !!! 

    # if you DO HAVE THE DB - probably not a bad idea to first backup it !!!
    # bash src/bash/qto/qto.sh -a backup-postgres-db
    # psql -d postgres -c "CREATE DATABASE dev_qto;"
    # drop it if it was there ...
    # psql -d postgres -c "DROP DATABASE dev_qto;"
    
    # create the db
    psql -d postgres -c "CREATE DATABASE dev_qto;"
    
    psql -d dev_qto < dat/mix/2020/2020-01/2020-01-11/sql/tst_qto/tst_qto.20200111_195947.full.dmp.sql

#### 4.7.2. Restore a database from db inserts file
This type of restore assumes you are 100% sure that the schema you took the db inserts backup from is the same as the schema you are applying the db inserts to, which should be the case when you are restoring data from the same qto version db. In qto we do not believe in migrations, which is a totally different discussion, but if you do not have the same schema you WILL HAVE errors and you ARE basically on your own, because you ended-up here by basically NOT taking regularly backups and not applying regular updates and that is YOUR fault and not the fault of the product instance owner you are getting the qto source from ....

    # if you DO HAVE THE DB - probably not a bad idea to first backup it !!!
    # bash src/bash/qto/qto.sh -a backup-postgres-db
    # psql -d postgres -c "CREATE DATABASE dev_qto;"
    # drop it if it was there ...
    # psql -d postgres -c "DROP DATABASE dev_qto;"
    
    
    bash src/bash/qto/qto.sh -a run-qto-db-ddl
    
    psql -d $postgres_app_db < \
    dat/mix/sql/pgsql/dbdumps/dev_qto/dev_qto.20180813_202202.insrt.dmp.sql

### 4.8. Restore a database table
You restore a database table by first running the pgsql scripts of the project database or ONLY for that table and than restoring the insert data from the table insert file.

    # re-apply the table ddl
    psql -d $postgres_app_db < src/sql/pgsql/dev_qto/13.create-table-requirements.sql

### 4.9. Create a full backup of an instance site from a satellite host
Once an instance site is up-and-running - it might be a good idea to create a full-backup of the site's instance data, software and configuration:
 - full backup the postgres - both DDL and DML's
 - ensure the cnf/env/&lt;&lt;env&gt;&gt;.json is part of the met/.&lt;&lt;env&gt;&gt;.qto meta list file
 - add the DML and DDL dump files to the met/.&lt;&lt;env&gt;&gt;.qto meta list file
 - create a relative or full package zip file
 - rename it to reflect the fact that this file contains a full backup file

    # full backup the postgres - both DDL and DML's
    ./src/bash/qto/qto.sh -a backup-postgres-db
    ./src/bash/qto/qto.sh -a backup-postgres-db-inserts
    #  ensure the cnf/env/<<env>>.json is part of the met/.<<env>>.qto meta list file 
    # vim met/.<<env>>.qto
    
    #  add the DML and DDL dump files to the met/.<<env>>.qto meta list file, for example :
    echo dat/mix/2020/2020-03/2020-03-22/sql/prd_qto/prd_qto.20200322_123449.full.dmp.sql >> met/.tst.qto
    echo dat/mix/2020/2020-03/2020-03-22/sql/prd_qto/prd_qto.20200322_123633.insrts.dmp.sql >> met/.prd.qto
    
    # create a relative or full package zip file
    ./src/bash/qto/qto.sh -a create-relative-package
    
    # rename it to reflect the fact that this file contains a full backup file
    mv -v ../qto.0.8.1.prd.20200322_122924.ip-10-0-44-231.rel.zip ../qto.0.8.1.prd-fb.20200322_122924.ip-10-0-44-231.rel.zip

### 4.10. Backup multiple ec2 servers from the satellite host
 The user@host notation in the $PRODUCT_OWNER part ensures that each product instance dir WILL have an unique path even if they are moved between the different qto hosts:

    #set the variables 
    export ssh_server=qto.fi
    export ssh_user=ubuntu
    export src_dir=/home/ubuntu/opt/qto # on the satellite host 
    export tgt_dir=/hos/opt/opt # on the ssh-server
    
    # copy all from src_dir on the ssh server to the tgt dir on the satellite host 
    rsync -e "ssh -l USERID -i ~/.ssh/id_rsa.aws-ec2.qto.prd" -av -r --partial --progress --human-readable \
    	--stats $ssh_user@$ssh_server:$src_dir $tgt_dir
    
    # and verify
    clear ; find /hos/opt/qto -type d -maxdepth 0|sort -nr| less
    
    # result could look something like 
    /hos/opt/qto/qto.0.8.3.dev.ysg@host-name
    /hos/opt/qto/qto.0.8.2.tst.ysg@host-name.20200331_210320
    /hos/opt/qto/qto.0.8.2.tst.ysg@host-name
    /hos/opt/qto/qto.0.8.2.tst.ubuntu@qto.fi
    /hos/opt/qto/qto.0.8.2.tst.ubuntu@csitea.net
    /hos/opt/qto/qto.0.8.2.prd.ysg@host-name.20200331_210300
    /hos/opt/qto/qto.0.8.2.prd.ysg@host-name
    /hos/opt/qto/qto.0.8.2.prd.ubuntu@qto.fi
    /hos/opt/qto/qto.0.8.2.prd.ubuntu@csitea.net
    /hos/opt/qto/qto.0.8.2.dev.ysg@host-name
    /hos/opt/qto/qto.0.8.2.dev.ubuntu@qto.fi
    /hos/opt/qto/qto.0.8.1.tst.ysg@qto.fi
    /hos/opt/qto/qto.0.8.1.tst.ubuntu@qto.fi
    /hos/opt/qto/qto.0.8.1.prd.ysg@qto.fi
    /hos/opt/qto/qto.0.8.1.prd.ubuntu@qto.fi
    /hos/opt/qto/qto.0.8.1.<EMAIL>
    

## 5. SHELL ACTIONS


    

### 5.1. Backup directories with rsync
You can backup whole directories from remote to local via ssh with the following setup and naming conventions:
on the local design for each remote a different product owner 

    export ssh_server=qto.fi
    export ssh_user=ubuntu
    export src_dir=/home/ubuntu/opt/qto
    export tgt_dir=/hos/opt/opt
    
    # incremental backuip
    rsync -e "ssh -l USERID -i ~/.ssh/id_rsa.aws-ec2.qto.prd" -av -r --partial --progress --human-readable \
    	--stats $ssh_user@$ssh_server:$src_dir $tgt_dir
    clear ; find /hos/opt/qto -type d -maxdepth 1|sort -nr| less
    
    # single backup
    rsync -e "ssh -l USERID -i ~/.ssh/id_rsa.aws-ec2.qto.prd" -av -r --partial --progress --human-readable \
    	--stats --delete-excluded ubuntu@$ssh_server:$src_dir $tgt_dir

### 5.2. Load xls sheet to db a doc table
To load xls issues to db and from db to txt files

    export do_truncate_tables=0 ; 
    clear ; perl src/perl/qto/script/qto.pl --do xls-to-db --tables requirements_doc
    
    # check that the rows where inserted
     psql -d dev_qto -c "SELECT * FROM requirements_doc;"

### 5.3. Run increase-date action
You track the issues of your projects by storing them into xls files in "daily" proj_txt dirs. 
Each time the day changes by running the increase-date action you will be able to clone the data of the previous date and start working on the current date.

    bash src/bash/qto/qto.sh -a increase-date

## 6. NGINX CONFIGURATION
This section contains 

    

### 6.1. The nginx provisioning
Anytime you change the ngix configuration and confirm that the change is beneficial for the overall setup you must add the change to the provisioning code - aka to the branch.

    # this is the main nginx.conf 
    cnf/nginx/etc/nginx/nginx.conf
    
    cnf/nginx/
    cnf/nginx/etc
    cnf/nginx/etc/nginx
    cnf/nginx/etc/nginx/sites-available
    cnf/nginx/etc/nginx/sites-available/tst.localhost.cnf
    cnf/nginx/etc/nginx/sites-available/prd.localhost.cnf
    cnf/nginx/etc/nginx/sites-available/dev.localhost.cnf
    cnf/nginx/etc/nginx/sites-available/%env%.http-site.conf
    cnf/nginx/etc/nginx/sites-available/%env%.https-site.conf
    
    
    

### 6.2. Applying new nginx configuration
To apply new configuration ensure the following: 
 - are you on a PRODUCTION server ?!!! What will happen if the configuration is totally messed -up 

    # sudo nginx -c /etc/nginx/nginx.conf -t
    # clear ; sudo service nginx restart ; sudo service nginx status
    
    # if all is ok make a backup
    sudo cp -v /etc/nginx/nginx.conf cnf/nginx/etc/nginx/nginx.conf.$(date "+%Y%m%d_%H%M%S").bak
    sudo cp -v /etc/nginx/nginx.conf cnf/nginx/etc/nginx/nginx.conf
    
    # if errors , troubleshoot 
    sudo journalctl -xe | less
    
    

### 6.3. Opening a monthly period
Each monthly period is basically a table with the naming convention monthly_issues_YYYYMM for example the monthly period for the year 2020 January will be the monthly_issues_202001. 
To create the table you need to basically copy the monthly_issues table to the one with the period at the end and replace the monthly issues with the monthly_issues_&lt;&lt;yyyymm&gt;&gt; string and run the table as follows.

    # copy the table
    cp -v src/sql/pgsql/qto/tables/03.create-table-monthly_issues.sql \ src/sql/pgsql/qto/tables/2020/2020-01/create-table-monthly_issues_202001.sql
    
    # source the env vars loading func  
    source lib/bash/funcs/export-json-section-vars.sh
    
    # load the env vars
    doExportJsonSectionVars cnf/env/dev.env.json '.env.db'
    psql -d dev_qto < src/sql/pgsql/qto/tables/2020/2020-01/create-table-monthly_issues_202001.sql
    
    # generate the sql for moving the data between the two tables
    bash src/tpl/psql-code-generator/psql-code-generator.sh tst_qto monthly_issues monthly_issues_202001
    

### 6.4. Closing a monthly period
Closing a monthly period would simply mean to ensure that all the done issues will be moved to the release_issues table an those marked with the '09-done' and '04-diss' status are removed AFTER they have been "copied" aka upserted to the yearly_issues_&lt;&lt;yyyyy&gt;&gt; and monthly_issues_&lt;&lt;yyyymm&gt;&gt; table of the previous month.

    source lib/bash/funcs/export-json-section-vars.sh
    doExportJsonSectionVars cnf/env/tst.env.json '.env.db'
    alias psql="PGPASSWORD=${postgres_sys_usr_admin_pw:-} psql -v -t -X -w -U ${postgres_sys_usr_admin:-} --port $postgres_rdbms_port --host $postgres_rdbms_host"
    psql -d tst_qto -c "delete from monthly_issues_202004 where issues_status_guid = 'e486d2d7-0789-4af2-8466-9b8c03743d85';"
    clear ; history | cut -c 8-

## 7. BULK ISSUES MANAGEMENT
By bulk issues management herewith is meant the bulk handling via sql to the issues items via the psql binary, which IS basically just moving data from table to table provided they have the same set of columns. 

    

### 7.1. Dirs naming conventions
The dir structure should be logical and a person navigating to a dir should almost understand what is to be find in thre by its name .. 

    

### 7.2. Publish the release issues
You publish the release issues by moving all the issues with '09-done' status to the release_issues table

    

## 8. NAMING CONVENTIONS


    

### 8.1. Naming conventions for directories
The dir structure should be logical and a person navigating to a dir should almost understand what is to be find in thre by its name .. 

    

### 8.2. Root Dirs naming conventions
The root dirs and named as follows:
bin - contains the produced binaries for the project
cnf - for the configuration
dat - for the data of the app
lib - for any external libraries used
src - for the source code of the actual projects and subprojects

    

## 9. SOURCE CODE MANAGEMENT
The qto is a derivative of the wrapp tool - this means that development and deployment process must be integrated into a single pipeline. 

    

### 9.1. Configure and use git ALWAYS by using ssh identities
You probably have access to different corporate and public git repositories. Use your personal ssh identity file you use in GitHub to push to the qto project. The following code snippet demonstrates how you could preserve your existing git configurations ( even on corporate / intra boxes ) , but use ALWAYS the personal identity to push to the qto...

    # create the company identity file
    ssh-keygen -t rsa -b 4096 -C "<EMAIL>"
    
    # save private key to ~/.ssh/id_rsa.corp, 
    cat ~/.ssh/id_rsa.corp.pub 
    
    # copy paste this string into your corp web ui security ssh keys
    
    # create your private identify file
    ssh-keygen -t rsa -b 4096 -C "<EMAIL>"
    # save private key to ~/.ssh/id_rsa.me, note the public key ~/.ssh/id_rsa.me.pub
    cat ~/.ssh/id_rsa.me.pub # copy paste this one into your githubs, private keys
    
    # set alias for the git command to avoid overtyping ...
    alias git='GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa.ysg " git'
    
    # clone a repo 
    git clone <EMAIL>:corp/project.git
    
    export git_msg="my commit msg with my corporate identity, explicitly provide author"
    git add --all ; git commit -m "$git_msg" --author "MeFirst MeLast <<EMAIL>>"
    git push 
    # and verify 
    clear ; git log --pretty --format='%h %ae %<(15)%an ::: %s

### 9.2. Aim for e2e traceability
Aim for traceability between user-stories, requirements, features and functionalities
Once the issues are defined and you start working on your own branch which is named by the issue-id aim to map one on one each test in your code with each listed requirement in qto.

    

### 9.3. Restart the application layer
Well just chain the both commands. 

    # start does actually re-start always 
    bash src/bash/qto/qto.sh -a mojo-morbo-start 

## 10. KNOWN ISSUES AND WORKAROUNDS


    

### 10.1. Morbo is stuck


    

#### 10.1.1. Symptoms
This one occurs quite often , especially when the application layer is restarted, but the server not 

    # the error msg is 
     [INFO ] 2018.09.14-10:23:14 EEST [qto][@host-name] [4426] running action :: mojo-morbo-start:doMojoMorboStart
    (Not all processes could be identified, non-owned process info
     will not be shown, you would have to be root to see it all.)
    tcp        0      0 0.0.0.0:3001            0.0.0.0:*               LISTEN      6034/qto
    tcp        0      0 0.0.0.0:3002            0.0.0.0:*               LISTEN      7626/qto
    Can't create listen socket: Address already in use at /usr/local/share/perl/5.26.0/Mojo/IOLoop.pm line 130.
     [INFO ] 2018.09.14-10:23:16 EEST [qto][@host-name] [4426] STOP FOR qto RUN with:
     [INFO ] 2018.09.14-10:23:16 EEST [qto][@host-name] [4426] STOP FOR qto RUN: 0 0 # = STOP MAIN = qto
    qto-dev ysg@host-name [Fri Sep 14 10:23:16] [/vagrant/opt/csitea/qto/qto.0.4.9.dev.ysg] $

#### 10.1.2. Probable root cause
This one occurs quite often , especially when the application layer is restarted, but the server not 

    # the error msg is 
     [INFO ] 2018.09.14-10:23:14 EEST [qto][@host-name] [4426] running action :: mojo-morbo-start:doMojoMorboStart
    (Not all processes could be identified, non-owned process info
     will not be shown, you would have to be root to see it all.)
    tcp        0      0 0.0.0.0:3001            0.0.0.0:*               LISTEN      6034/qto
    tcp        0      0 0.0.0.0:3002            0.0.0.0:*               LISTEN      7626/qto
    Can't create listen socket: Address already in use at /usr/local/share/perl/5.26.0/Mojo/IOLoop.pm line 130.
     [INFO ] 2018.09.14-10:23:16 EEST [qto][@host-name] [4426] STOP FOR qto RUN with:
     [INFO ] 2018.09.14-10:23:16 EEST [qto][@host-name] [4426] STOP FOR qto RUN: 0 0 # = STOP MAIN = qto
    qto-dev ysg@host-name [Fri Sep 14 10:23:16] [/vagrant/opt/csitea/qto/qto.0.4.9.dev.ysg] $

#### 10.1.3. Known solution and workaround
List the running perl processes which run the morbo and kill the instances

    ps -ef | grep -i perl
    
    # be carefull what to kill 
    kill -9 <<proc-I-know-is-the-one-to-kill>>

### 10.2. Nginx presents http 502
The root cause of this error is not exactly known ... Just restart the via the hypnotoad restart shell action ...

    # in the dev tmux window !!!
    cd ~/opt/qto/qto.0.8.3.dev.ubuntu@qto.fi
    bash src/bash/qto/qto.sh -a mojo-hypnotoad-start & # restart 
    
    # in the tst tmux window !!!
    cd ~/opt/qto/qto.0.8.3.tst.ubuntu@qto.fi
    bash src/bash/qto/qto.sh -a mojo-hypnotoad-start &
    
    # in the prd tmux window !!!
    cd ~/opt/qto/qto.0.8.3.prd.ubuntu@qto.fi
    bash src/bash/qto/qto.sh -a mojo-hypnotoad-start &
    
    # detach from the tmux session
    

## 11. LINUX SYSTEM ADMINISTRATION


    

### 11.1. Check open file descriptors
To check qto opened file descriptors issue the following command:
For further reading: 
https://www.cyberciti.biz/faq/linux-increase-the-maximum-number-of-open-files/
https://mojolicious.org/perldoc/Mojo/Server/Hypnotoad#WORKER-SIGNALS
https://www.cyberciti.biz/tips/linux-procfs-file-descriptors.html

    while read -r p ; do lsof -a -p $p ; done < <(ps aux | grep perl|awk '{print $2}') | less

## 12. SECURITY


    

### 12.1. Check the login events on a qto instance
To heck the login events on a qto instance issue the following command from the product instance dir.

    grep -nHi 'login ' dat/log/qto.2020.04*

<file_sep># src/bash/qto/funcs/%act%.%deliverable_type%.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add %full_func% comments ...
# ---------------------------------------------------------
%full_func%(){

	do_log "DEBUG START %full_func%"
	
	cat doc/txt/qto/%deliverable_type%s/%act%.%deliverable_type%.txt
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!

	do_log "DEBUG STOP  %full_func%"
}
# eof func %full_func%


# eof file: src/bash/qto/funcs/%act%.%deliverable_type%.sh
<file_sep>

do_chk_install_chromium_headless(){
   which chromium-browser 2>/dev/null || {
      sudo apt-get update
      sudo apt-get install -y software-properties-common
      sudo apt-get install -y chromium-browser 
      sudo apt-get update

      wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
      sudo dpkg -i google-chrome-stable_current_amd64.deb
      apt --fix-broken install
   }
}
<file_sep>   INSERT INTO app_items
   ( name , description)

   select tablename as name , tablename as description 
   FROM pg_tables 
   where 1=1
   and schemaname='public'
   ORDER BY name
   ON CONFLICT (id) DO UPDATE SET
   guid = excluded.guid, id = excluded.id, prio = excluded.prio, name = excluded.name, description = excluded.description, update_time = excluded.update_time;

<file_sep>#!/bin/bash

do_run_qto_db_ddl(){

   test -z "${PROJ_INSTANCE_DIR-}" && export PROJ_INSTANCE_DIR="$PRODUCT_DIR"
   test -z ${PROJ_CONF_FILE:-} && export PROJ_CONF_FILE="$PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json"
   do_export_json_section_vars $PROJ_CONF_FILE '.env.db'
   do_log "INFO using PROJ_INSTANCE_DIR: $PROJ_INSTANCE_DIR" ; do_log "INFO using PROJ_CONF_FILE: $PROJ_CONF_FILE"
   sleep 3;

   # 00 create the db
   tmp_log_file="$tmp_dir/.$$.log"
	pgsql_scripts_dir="$PRODUCT_DIR/src/sql/pgsql/qto"
   sql_script="$pgsql_scripts_dir/00.create-db.pgsql"

   PGPASSWORD="${postgres_sys_usr_admin_pw:-}" psql -v -q -t -X -w -U "${postgres_sys_usr_admin:-}" \
      -h $postgres_rdbms_host -p $postgres_rdbms_port -v ON_ERROR_STOP=1 \
		-v postgres_app_db="${postgres_app_db:-}" \
      -f "$sql_script" postgres > "$tmp_log_file" 2>&1
   ret=$?
   cat "$tmp_log_file" ; cat "$tmp_log_file" >> $log_file # show it and save it
   test $ret -ne 0 && {
      sleep 3 ; do_exit 1 "pid: $$ psql ret $ret - failed to run sql_script: $sql_script !!!" ; break ;
   }

   doRunPgsqlScripts
   # 01 create / modify the app user
   sql_script="$pgsql_scripts_dir/01.create-qto-app-user.pgsql"
   #sql_script="$pgsql_scripts_dir/02.alter-qto-app-user.pgsql"
   PGPASSWORD="${postgres_sys_usr_admin_pw:-}" psql -q -t -X -w -U "${postgres_sys_usr_admin:-}" \
      -h $postgres_rdbms_host -p $postgres_rdbms_port -v ON_ERROR_STOP=1 \
      -v postgres_app_usr="${postgres_app_usr:-}" \
      -v postgres_app_usr_pw="${postgres_app_usr_pw:-}" \
      -v postgres_app_db="${postgres_app_db:-}" \
      -f "$sql_script" "${postgres_app_db:-}" > "$tmp_log_file" 2>&1
   ret=$?

   cat "$tmp_log_file" ; cat "$tmp_log_file" >> $log_file # show it and save it
   test $ret -ne 0 && {
      sleep 3 ; do_exit 1 "pid: $$ psql ret $ret - failed to run sql_script: $sql_script !!!";
   }
   PGPASSWORD="${postgres_sys_usr_admin_pw:-}" psql -q -t -X -w -U "${postgres_sys_usr_admin:-}" \
      -h $postgres_rdbms_host -p $postgres_rdbms_port -d postgres -v ON_ERROR_STOP=1 -c \
      "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO $postgres_app_usr;
	  GRANT CONNECT ON DATABASE $postgres_app_db TO $postgres_app_usr;
	  GRANT SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA public TO $postgres_app_usr;
      GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO $postgres_app_usr;"

}
<file_sep>#  FEATURES AND FUNCTIONALITIES 
* [1. INTRO](#1-intro)
  * [1.1. PURPOSE](#11-purpose)
  * [1.2. AUDIENCE](#12-audience)
  * [1.3. RELATED DOCUMENTATION](#13-related-documentation)
  * [1.4. RELATED PROCESS AND WAY OF WORKING](#14-related-process-and-way-of-working)
* [2. DEPLOYABILITY](#2-deployability)
  * [2.1. FULL DEPLOYMENT IN LESS THAN AN HOUR](#21-full-deployment-in-less-than-an-hour)
  * [2.2. EASY DOCKER BASED FULL DEPLOYMENT ( DEPRECATING )](#22-easy-docker-based-full-deployment-(-deprecating-))
    * [2.2.1. New version deployment by simple unzip](#221-new-version-deployment-by-simple-unzip)
    * [2.2.2. Oneliner for prerequisite binaries check](#222-oneliner-for-prerequisite-binaries-check)
    * [2.2.3. Installation documentation](#223-installation-documentation)
  * [2.3. A FULL APPLICATION CLONE IN LESS THAN 5 MINUTES](#23-a-full-application-clone-in-less-than-5-minutes)
    * [2.3.1. Shell script for postgres db creation](#231-shell-script-for-postgres-db-creation)
    * [2.3.2. One liner for single restore and / or load](#232-one-liner-for-single-restore-and-/-or-load)
* [3. USER-FRIENDLINESS](#3-user-friendliness)
  * [3.1. ONELINER SHELL CALLS](#31-oneliner-shell-calls)
    * [3.1.1. Database recreation and DDL scripts run one-liners](#311-database-recreation-and-ddl-scripts-run-one-liners)
    * [3.1.2. Testing one-liner call](#312-testing-one-liner-call)
  * [3.2. TABLES LOAD VIA A SINGLE ONE-LINER](#32-tables-load-via-a-single-one-liner)
    * [3.2.1. Test messages user](#321-test-messages-user)
* [4. STABILITY AND RELIABILITY](#4-stability-and-reliability)
  * [4.1. ABSENCE OF APPLICATION CRASHING](#41-absence-of-application-crashing)
  * [4.2. DAILY BACKUPS](#42-daily-backups)
  * [4.3. LOGGING](#43-logging)
  * [4.4. FULL BACKUP TO THE CLOUD IN LESS THAN 5 MINUTES](#44-full-backup-to-the-cloud-in-less-than-5-minutes)
  * [4.5. STABILITY BASED ON ACTUAL RUNNING IN THE CLOUD SINCE 2019-01-01](#45-stability-based-on-actual-running-in-the-cloud-since-2019-01-01)
* [5. SCALABILITY](#5-scalability)
  * [5.1. FEATURE SCALABILITY](#51-feature-scalability)
  * [5.2. MULTI-INSTANCE SCALABILITY](#52-multi-instance-scalability)
  * [5.3. DOCUMENTATION GENERATION AND DATA EXPORT](#53-documentation-generation-and-data-export)
  * [5.4. PROJECTS DATABASES SCALABILITY](#54-projects-databases-scalability)
* [6. PERFORMANCE](#6-performance)
  * [6.1. PAGE LOAD TIMES](#61-page-load-times)
    * [6.1.1. Page load times](#611-page-load-times)
    * [6.1.2. Server requirements for performance](#612-server-requirements-for-performance)
    * [6.1.3. Performance on mobile devices ](#613-performance-on-mobile-devices-)
    * [6.1.4. Loaded pages controls performance](#614-loaded-pages-controls-performance)
  * [6.2. AJAX CALLS TO BACK-END](#62-ajax-calls-to-back-end)
    * [6.2.1. Metadata caching via Redis](#621-metadata-caching-via-redis)
    * [6.2.2. Foreign Key data for Drop Down controls caching](#622-foreign-key-data-for-drop-down-controls-caching)
  * [6.3. LOGIN, LOGOUT](#63-login-logout)
  * [6.4. CRUDS OPERATIONS](#64-cruds-operations)
* [7. DEPLOYABILITY](#7-deployability)
  * [7.1. NEW HOSTS BINARY PROVISIONING](#71-new-hosts-binary-provisioning)
  * [7.2. ONELINER FOR VERSION CHANGE ON PROVISIONED HOST](#72-oneliner-for-version-change-on-provisioned-host)
  * [7.3. ONELINER FOR ENVIRONMENT CHANGE](#73-oneliner-for-environment-change)
* [8. WEB USER INTERFACE FEATURES AND FUNCTIONALITIES](#8-web-user-interface-features-and-functionalities)
  * [8.1. SINGLE CODE BASE FOR ALL DEVICE FORMATS](#81-single-code-base-for-all-device-formats)
  * [8.2. GLOBAL UI ELEMENTS FOR ALL THE PAGES](#82-global-ui-elements-for-all-the-pages)
    * [8.2.1. The global and quick search textbox - the omnibox](#821-the-global-and-quick-search-textbox--the-omnibox)
    * [8.2.2. The Left Menu](#822-the-left-menu)
  * [8.3. LIST AND LIST-MY PAGE FEATURES AND FUNCTIONALITIES](#83-list-and-list-my-page-features-and-functionalities)
    * [8.3.1. List page performance](#831-list-page-performance)
    * [8.3.2. Navigating the list page](#832-navigating-the-list-page)
    * [8.3.3. Filtering the result set from a single item](#833-filtering-the-result-set-from-a-single-item)
      * [8.3.3.1. Filtering the result with the "with" url parameter](#8331-filtering-the-result-with-the-"with"-url-parameter)
      * [8.3.3.2. Filtering the result with the "like-by" and "like-val" url parameters](#8332-filtering-the-result-with-the-"like-by"-and-"like-val"-url-parameters)
    * [8.3.4. Managing items - full CRUD](#834-managing-items--full-crud)
    * [8.3.5. Managing FK values](#835-managing-fk-values)
      * [8.3.5.1. CRUD via the smart grid](#8351-crud-via-the-smart-grid)
      * [8.3.5.2. CRUD via the generic edit form](#8352-crud-via-the-generic-edit-form)
    * [8.3.6. Quick search and filtering items](#836-quick-search-and-filtering-items)
    * [8.3.7. Global text search from the view page](#837-global-text-search-from-the-view-page)
      * [8.3.7.1. Visual indication for ok cases](#8371-visual-indication-for-ok-cases)
    * [8.3.8. Visual indication](#838-visual-indication)
      * [8.3.8.1. Visual indication when no data is received on correct request](#8381-visual-indication-when-no-data-is-received-on-correct-request)
      * [8.3.8.2. Visual indication on erroneous requests](#8382-visual-indication-on-erroneous-requests)
    * [8.3.9. Items interlinking](#839-items-interlinking)
    * [8.3.10. Print to pdf](#8310-print-to-pdf)
  * [8.4. VIEW PAGE FEATURES](#84-view-page-features)
    * [8.4.1. Viewing documents](#841-viewing-documents)
      * [8.4.1.1. OK load for a view doc page](#8411-ok-load-for-a-view-doc-page)
      * [8.4.1.2. NOK load for view doc page](#8412-nok-load-for-view-doc-page)
      * [8.4.1.3. NOK load for a view doc page](#8413-nok-load-for-a-view-doc-page)
    * [8.4.2.  Managing items ( beta )](#842--managing-items-(-beta-))
      * [8.4.2.1. Add an item in the doc view page UI ( beta)](#8421-add-an-item-in-the-doc-view-page-ui-(-beta))
      * [8.4.2.2. Update item ( gamma )](#8422-update-item-(-gamma-))
      * [8.4.2.3. Delete item ( gamma )](#8423-delete-item-(-gamma-))
    * [8.4.3. Printing view doc documents](#843-printing-view-doc-documents)
* [9. SECURITY](#9-security)
  * [9.1. AUTHENTICATION](#91-authentication)
    * [9.1.1. Non-athentication mode](#911-non-athentication-mode)
    * [9.1.2. Simple native authentication mode](#912-simple-native-authentication-mode)
  * [9.2. AUTHORISATION](#92-authorisation)
    * [9.2.1. Roles based access control list](#921-roles-based-access-control-list)
* [10. DOCUMENTATION](#10-documentation)
  * [10.1. DOCUMENTATION SET](#101-documentation-set)
  * [10.2. DOCUMENTATION FORMATS](#102-documentation-formats)
  * [10.3. DOCUMENTATION AND CODE BASE SYNCHRONIZATION](#103-documentation-and-code-base-synchronization)




    

## 1. INTRO


    

### 1.1. Purpose
The purpose of this document is to present the features and functionalities set to the qto application for this current version.

    

### 1.2. Audience
This document could be of interest for any potential and actual users of the application. Developers and Architects working on the application MUST read and understand this document at least to the extend of their own contribution for it. 

    

### 1.3. Related documentation
This document is part of the qto application documentation-set, which contains the following documents:
 - readme - the initial readme file of the project
 - enduser_guide_doc - the end users guide - mostly ui usage and concepts
 - concepts_doc - contains the concepts of the application
 - userstories_doc - the collection of userstories used to describe "what is desired"
 - system_guide_doc - architecture and System description
 - devops_guide_doc - a guide for the developers and devops operators
 - installation_guide_doc - a guide for installation of the application
 - features_doc - description of the current features and functionalities
 - requirements_doc - description of the Requirements for the application


All the documents are updated and redistributed in combination of the current version of the application and could be found under the following directories:
doc/md
doc/pdf
according to the file format used for the documentation storage.

    

### 1.4. Related process and way of working
Read this document carefully - it IS the contract between you AND the qto instance of your instance, such there be any inconsistencies between the promised features and functionalities in this document and the ACTUAL behaviour of your instance create an issue to the instance owner, describing clearly what the inconsistencies are.
If you are the instance owner, and the issue has clearly arisen from the source code of qto instance you derived your instance from create an issue to the source qto instance owner into his/her own qto instance.

    

## 2. DEPLOYABILITY
The qto is easily deployable on any Unix like OS. Windows family based OS'es are explicitly out of the scope of the qto tool. Any qto instance should be configurable as easily as possible for the version it has.  

    

### 2.1. Full deployment in less than an hour
The full System is ready for use in a "blank" Unix-like OS host in less than an hour.
The installations instructions are done for Ubuntu 18.04 LTS, yet should you feel confortable with other Linux distros or even BSD Unix you should be able to complete it in less than 2 hours. 

    

### 2.2. Easy docker based full deployment ( deprecating )
This feature is being deprecated ... yet you could quickly re-implement it by changing the Dockerfiles versions .. By following the installation instructions in the installations_doc you can deploy on any docker running Unix-like OS the qto application running on a docker and Ubuntu 18.04 LTS with initially loaded database and data. 

    

#### 2.2.1. New version deployment by simple unzip
The qto tool could be deployed by a simply unzip of the full package, which must have all of the documentation and scripts to provide assistance for the setup and the configuration of the tool. 

    

#### 2.2.2. Oneliner for prerequisite binaries check
All the binaries which are required for the running of the tool must be checked by a user-friendly binaries prerequisites check script

    

#### 2.2.3. Installation documentation
The installation of the required mysql and postgres db must be documented in the DevOps guide, which should have both markdown and pdf versions in the doc directory of the deployment package. 

    

### 2.3. A full application clone in less than 5 minutes
A DevOps operator is able to perform an application clone of the qto application in less than 5 minutes. 

    

#### 2.3.1. Shell script for postgres db creation
The creation of the postgres database is doable via a single shell call. 

    

#### 2.3.2. One liner for single restore and / or load
A qto db clone can be loaded via a single oneliner.

    

## 3. USER-FRIENDLINESS
The interaction with each endpoint and interface of an application instance is implemented to be as user-friendly as possible. 
As abstract as it may sound the tool is multi-dimensionally and vertically integrated regarding the questions what, how and why towards a new person interacting with the tool by the usage of code comments, links from the documentations and uuids to be used for simple grepping from the docs till the source code. 

    

### 3.1. Oneliner shell calls
The interaction of the application on the shell should be designed and implemented so that most of the features and bigger entry points should be accessible via one-liners on the shell - for example the testers should be able to lunch all the unit-tests via a single one line call. The integration tests should be triggerable via single oneline call. 

    

#### 3.1.1. Database recreation and DDL scripts run one-liners
The developers should be able to create the database via a single oneline call 

    

#### 3.1.2. Testing one-liner call
The testers and the developers is able to trigger all the unit or integration tests via a single one-line call. 

    

### 3.2. Tables load via a single one-liner
The developers should be able to load one or many tables to the database via a single oneline call.

    

#### 3.2.1. Test messages user
Each test obeys the following convention:
 - short message as descriptive within the context as possible - what is being tested
- a short technical example of the generated entry being tested ( for example a dynamic url )
- a uuid to search for from the Feature document what exactly is being tested within the context of the features description. 

    

## 4. STABILITY AND RELIABILITY


    

### 4.1. Absence of application crashing
Qto is a vertically integrated application - from the OS till the client side libraries, which has been designed from the ground up to have stable architecture component wise and to rely on the best frameworks (Mojolicious, VueJS), libraries and tools out there, without creating too much interdependencies, thus the qto crashes experienced during the last years of operation have been extremely rare.

    

### 4.2. Daily backups
Daily backups are taken after the first shell action, run for the day, the daily backups oneliner could be scheduled via crontab as well. 

    

### 4.3. Logging
The application supports fully configurable audit logging to both console ( STDOUT, STDERR ) and file.

    

### 4.4. Full backup to the cloud in less than 5 minutes
A full backup for the data for the qto and/or another project database is doable in less than 5 minutes. 

    

### 4.5. Stability based on actual running in the cloud since 2019-01-01
The main qto application instance has been up-and-running since the beginning of 2019 with receiving new versions in an average of 2 weeks per sprint.

    

## 5. SCALABILITY


    

### 5.1. Feature scalability
The addition of new features is scalable, as almost all of the components have been implemented according to the SOLID principle. 

    

### 5.2. Multi-instance scalability
You can operated multiple versions of the qto on the same host, which will not interrupt each other because of the configurability of the application.

    

### 5.3. Documentation generation and data export
Single shell actions exist for : 
 - configurable pdf documentation generation
 - configurable mdf documentation generation
 - configurable microsoft docx documentation generation
 - configurable xls tables export

    

### 5.4. Projects databases scalability
Each instance of the qto application can connect via tcp to multiple postgres databases running on the same db host configured in the instance configuration file. 

    

## 6. PERFORMANCE


    

### 6.1. Page load times
Although the qto has not been explicitly designed for mobile devices the page load times on higher end phones on 4G networks are comparable with the upper 20% of the pages in the web, while running both the application layer and the db on a single aws instance based in Ireland ...

    

#### 6.1.1. Page load times
The qto has been operated on quite modest hardware ( check the second cheapest amazon ec2 instances for reference ), yet the page load times vary from 0.3 till 0.6 seconds for the smaller pages and up till 1.5 seconds for the pages having more than 2000 items ...
List pages with 7-10 items load in 0.3-0.35 seconds.
View Pages with one image and less than 7 pages load in 0.3-0.45 seconds, View pages with 20 pages might take up till 0.8 seconds to load.

    

#### 6.1.2. Server requirements for performance
The qto has been operated on quite modest hardware ( check the second cheapest amazon ec2 instances for reference ), yet the performance of the application has been amazingly good, considered that both the application server and the databases are deployed on the ec2 instance.

    

#### 6.1.3. Performance on mobile devices 
Qto is NOT explicitly optimised for mobile use ( how old fashioned is that ?!), yet it has been from the ground-up designed in way allowing this optimisation to occur at a later point of time of the development of the application. 

    

#### 6.1.4. Loaded pages controls performance
Once a page is loaded, more than 99% of the time the controls response quickly and effortlessly on different keyboard or mouse events.

    

### 6.2. Ajax calls to back-end
Each back-end update from the UI takes no longer than 0.2 s. in a non-stressed qto instance, thus the look and feel of an qto instance is more like a desktop app and less like web app.

    

#### 6.2.1. Metadata caching via Redis
All the meta data of the application during each request is fetched from Redis ( and can be reload into Redis by loading the app_items or the app_item_attributes item in the browser)

    

#### 6.2.2. Foreign Key data for Drop Down controls caching
The drop down controls which fetch the human readable data with the foreign keys of a normalised item table is cached to the localStorage. The localStorage is cleared once the users access the login page. 
Thus a qto DevOps engineer could both easily modilfy the data for the dropdowns ( for example different statuses and their codes without having to restart the whole application layer), but the users get a faster and non-blinking list page loads once they have loaded the list page with the specific cached dropdowns data once.

    

### 6.3. Login, Logout
Every login and logout operation completes in less than 0.3 seconds in modern network environments.

    

### 6.4. CRUDs operations
The Create, Update, Delete and Search operations with their respective roundtrips are 99% of the times extremely fast - less than 0.1.

    

## 7. DEPLOYABILITY


    

### 7.1. New hosts binary provisioning
You can spawn new instances in the cloud from a client having the src code and needed terraform binary 

    

### 7.2. Oneliner for version change on provisioned host
You could create a new instance of the qto having different version ( which becomes automatically a dev environment ) by issuing the following command: 

    bash src/bash/qto/qto.sh -a to-ver=0.8.2

### 7.3. Oneliner for environment change
You could change the environment type of your current instance by issuing the following command:

    bash src/bash/qto/qto.sh -a to-env=tst

## 8. WEB USER INTERFACE FEATURES AND FUNCTIONALITIES


    

### 8.1. Single code base for all device formats
Although qto has not been explicitly designed for mobile phones and / or tablets it renders well on both high end mobile phones, and tablets over 4G networks. 

    

### 8.2. Global UI elements for all the pages
The global UI elements are accessible from each page

    

#### 8.2.1. The global and quick search textbox - the omnibox
The users can quick filter the content of any page - lists ( including the drop downs chosen values), reports and view docs by just typing strings to filter by in the omnibox in the top bar WITHOUT hitting enter or clicking on the search button. 
After hitting enter or clicking on the search button the application performs global search from the project database and redirects to the search page with the actual result set from the search. Most of the pages are done so that hitting the back button on the browser goes directly to the previous view.
A global keyboard shortcut - the "/" forward slash can be used to focus the omnibox from any non editable control on the page.

    

#### 8.2.2. The Left Menu
The left menu is accessible from all the pages. The users can navigate from a folders tree like structure from the left menu to different pages in the application.
The left menu contains information on the current instance of the application and a logout button, which clears out the user session data ( both session and local storage data).

    

### 8.3. List and list-my page features and functionalities
The list page is simply a slice/subset of the data from ANY postgres table filtered on any criteria defined in the url of the browser.
The list-my route works exactly as the list page except for the fact that the retrieved subset is restricted to the data of a table, which is assigned to the currently logged in user by means of a foreign key to the app_users table.
An example url of a list page :
https://qto.fi/qto/list/yearly_issues_2020?&oa=prio&pg-size=5
An example url of a list-my page:
https://qto.fi/qto/list-my/yearly_issues_2020?&oa=prio&pg-size=5

    

#### 8.3.1. List page performance
The full execution time of any crud operation ( create, update, delete, search) from the end-user of the UI point of view is less than 0.5 seconds usually right around 0.4 s for faster servers.
Dropdown controls' data is cached in the browsers' local storage thus ones a page is accessed during an user session ( as the storage is cleared during login ), the next retrieval of their data is about 0.90-0.12 seconds faster - and their rendering is not actually noticeable.

    

#### 8.3.2. Navigating the list page
After the load of the list page the user can quickly cycle trough all the element of the page with the tab key on the keyboard. Focus on the search 

    

#### 8.3.3. Filtering the result set from a single item
The qto list page uses a SQL like syntax to translate the url parameters of the list page 

    

##### 8.3.3.1. Filtering the result with the "with" url parameter
The naming conventions is as follows: 
with=&lt;&lt;attribute-name&gt;&gt;-&lt;&lt;operator&gt;&gt;-&lt;&lt;value&gt;

For example:
https://qto.fi:442/qto/list/ideas?&with=owner-eq-jrs

This translates into "where &lt;&lt;attribute-name&gt;&gt; = '&lt;&lt;attribute-value&gt;&gt;' in the sql query to the database.

    

##### 8.3.3.2. Filtering the result with the "like-by" and "like-val" url parameters
The naming conventions is as follows: 
with=&lt;&lt;attribute-name&gt;&gt;-&lt;&lt;operator&gt;&gt;-&lt;&lt;value&gt;

For example:
https://qto.fi:442/qto/list/ideas?&with=owner-eq-jrs

    

#### 8.3.4. Managing items - full CRUD
The System provides the needed UI interfaces to Create , Update , Delete and Search items in the database. DateDate values could be presented with calendar controls.

    

#### 8.3.5. Managing FK values
The Foreign key values could be managed via drop box ui controls. For now the fetch is performed on the "name" attribute from the PK table, based on &lt;&lt;pk_table&gt;&gt;_guid naming convention in the FK column of the items.

    

##### 8.3.5.1. CRUD via the smart grid
The Create, Update, Delete operations could be performed by simply typing the new values in the smart grid - or picking the values for the drop downs or the calendar controls.

    

##### 8.3.5.2. CRUD via the generic edit form
The Create, Update, Delete operations could be performed by clicking in the edit button of the smart grid and editing the text in the modal dialog form.

    

#### 8.3.6. Quick search and filtering items
The user can quickly filter the items from the presented listing by typing in the omnisearch box ... The System will shrink the table so that only the rows having the string in the search omnibox will be presented. Once the string is deleted from the search omnibox the table data is restored to its original state.

    

#### 8.3.7. Global text search from the view page
The user can initiate a global text search from the view page, by simply typing on the global search text box and hitting enter or clicking on the global search button. Should the search yield results they will be displayed in the search page, thus the user could quickly return back in the exact same spot of the document by clicking the back button.

    

##### 8.3.7.1. Visual indication for ok cases
In the ok cases no msgs are presented, but just the requested data shown.

    

#### 8.3.8. Visual indication
The Systems does not present any ok messages for the operation of the list page, only errors are presented clearly on the top of the page ( for example when one tries to update a string value into a cell with column accepting only integer values ... )

    

##### 8.3.8.1. Visual indication when no data is received on correct request
Should the request in terms of url parameters be correct a warning msg at the bottom of the page in grey colour is presented

    

##### 8.3.8.2. Visual indication on erroneous requests
Should the request be erroneous - un existing, tables , columns , ds , wrong syntax etc. an error of the top of the page is presented and the full technical error is displayed at the bottom of the page. 

    

#### 8.3.9. Items interlinking
The users can link to any items by simply typing &lt;&lt;item&gt;&gt;-&lt;&lt;id&gt;&gt; in the description. For example requirements_doc-4

    

#### 8.3.10. Print to pdf
You can print any of the queries from the list page by adding / changing the as url parameter from as=grid to as=print-table. Use the browser print to pdf feature to save the listing page into a pdf file.

    

### 8.4. View page features


    

#### 8.4.1. Viewing documents
The "view doc" page presents a single "doc" table from any qto db instance having the nested set api in it. The nested set api means simply that the table has the additional attributes needed for the data to be presented in hierarchical format aka the view doc format, which is the format of this type of document you are reading right now.

    

##### 8.4.1.1. OK load for a view doc page
Should a proper url format by requested from the browser a simple view doc page is presented with no msgs - the left menu will present the meta data of the project menu and the right menu will present the structure of the document.

    

##### 8.4.1.2. NOK load for view doc page
Should the user request an un existing view doc page and error snackbar pop-up message appears on the top of the page. The error message can be pinned for easier copy paste or snapshot taking

    

##### 8.4.1.3. NOK load for a view doc page
Should the browser request a non-existing table or erroneous url parameters from the application layer the system will present the error in a snackbar at the top of the page. The snackbar can be freezed by clicking on the top right corner of it. 

    

#### 8.4.2.  Managing items ( beta )
The Qto application provides the needed UI interfaces to Create , Update , Delete items in the view documents UI for the users having the privileges for those actions.
This feature is in beta mode as it certain rare cases it might not work as expected ( and those are recognised as separate issues and being worked out ). The easiest way to work around in case of a bug is found is to reload the whole page.

    

##### 8.4.2.1. Add an item in the doc view page UI ( beta)
Users with the write privileges for the document can add an item in the doc view page just by right-clicking on the title and selecting one of the 3 options:
 - add sibling node - add an item which is on the same level in the hierarchy 
 - add parent node - add an item which is on 1 level up in the hierarchy
 - add child node - add and item which is on 1 level below in the hierarchy
The new item appears straight after the origin title it was requested from.
This feature is in beta mode as it certain rare cases it might not work as expected ( and those are recognised as separate issues and being worked out ). The easiest way to work around in case of a bug is found is to reload the whole page.

    

##### 8.4.2.2. Update item ( gamma )
You can:
- update item title content
- update item description
- update item src code if visible ( you can make it visible by adding any non space content to it in the list page by right-clicking the item number and choosing open in list )
This feature is in gamma mode as it certain rare cases it might not work as expected ( and those are recognised as separate issues and being worked out ). The easiest way to work around in case of a bug is found is to reload the whole page.

    

##### 8.4.2.3. Delete item ( gamma )
You can right-click on an item and choose the remove node from the right-click men.
This feature is in gamma mode as it certain rare cases it might not work as expected ( and those are recognised as separate issues and being worked out ). The easiest way to work around in case of a bug is found is to reload the whole page.

    

#### 8.4.3. Printing view doc documents
You can print ANY branch including the whole documents by right-clicking the branch or node and choosing "print-preview" and than use the browser's print functionality.

    

## 9. SECURITY


    

### 9.1. Authentication
The qto application supports the following 2 modes of security: 
 - non authentication mode
 - simple native authentication mode 

    

#### 9.1.1. Non-athentication mode
Any qto instance supports a non-authentication mode - that is all users having http access could perform all the actions on the UI without restrictions

    

#### 9.1.2. Simple native authentication mode
A qto instance running under the simple native authentication mode stores the user credentials in the instance db. The passwords are encrypted via the using the Blowfish-based Unix crypt() hash function, known as "bcrypt" encrypting mechanism. 

    

### 9.2. Authorisation
Only the SysAdmin of the System can add basic authentication and simple native mode users, thus regular users can see only their own credentials.

    

#### 9.2.1. Roles based access control list
The qto application has a centralised app_roles based access control list accessible from the following url: 
&lt;&lt;proj-db&gt;&gt;/list/app_items_roles_permissions. 
This list will define who ( which app_roles ) , can or cannot do what and why - the description field. The actual application of this Access Control List has NOT been yet implemented ...

    

## 10. DOCUMENTATION


    

### 10.1. Documentation set
Each running instance has the following documentation set :
 - ReadMe
 - Features and Functionalities doc
 - End User Guide
 - DevOps Guide
 - SystemGuide
 - UserStories document
 - Installation and Configuration Guide
In both "native qto format" and md file format in the doc/md directory of the project.

    

### 10.2. Documentation formats
The qto documentation is available in both md and pdf formats. 

    

### 10.3. Documentation and code base synchronization
Each running instance has it's required documentation set up-to-date. No undocumented or hidden features are allowed. Should any be missing or misreported a new issue must be created to correct those with top priority. 

    

<file_sep>doSetupPyVEnv(){

   which pip3 2>/dev/null || { 
		if [[ "$OSTYPE" == "darwin"* ]]; then
			brew install python3; brew postinstall python3 ; fi
		if [ "$(grep -Ei 'debian|ubuntu|mint' /etc/*release)" ]; then
			sudo apt update; sudo apt install -y python3-pip python3-venv; fi

		which pip3 2>/dev/null || { 
			echo >&2 "The pip3 binary is missing ! Install it manually ! Aborting ..."; exit 1; 
		}
	}

   pip3 install virtualenv ; python3 -m venv ./venv
   pip3 install -r $PRODUCT_DIR/cnf/bin/python/requirements.txt
	sleep 1 ; do_flush_screen 
   installed_python_modules=$(cat $PRODUCT_DIR/cnf/python/requirements.txt)

   cat << EOF
   attempted to install the following python modules: 
		$installed_python_modules
   listed in the following file: 
		 $PRODUCT_DIR/cnf/bin/python/requirements.txt
  
	# to activate the virtual environment
	cd $PRODUCT_DIR
	source ./venv/bin/activate

   # you MIGHT have to 
   pip3 install -r $PRODUCT_DIR/cnf/bin/python/requirements.txt

   # to exit the venv, run the following cmd:
   deactivate

EOF
}
<file_sep>-- run as the postgres user
-- \c postgres
CREATE USER usrqtoadmin WITH SUPERUSER CREATEROLE CREATEDB REPLICATION BYPASSRLS PASSWORD '<PASSWORD>';
-- alias psql="PGPASSWORD=${postgres_sys_usr_admin_pw:-} psql -v -q -t -X -w -U ${postgres_sys_usr_admin:-}"
-- psql -d dev_qto < src/sql/pgsql/list-table-priviledges.sql | less
-- \c dev_qto
ALTER ROLE usrqtoapp WITH PASSWORD '<PASSWORD>' LOGIN ;

GRANT SELECT ON ALL TABLES IN SCHEMA PUBLIC TO usrqtoapp;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO usrqtoapp;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO usrqtoapp;

SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolconnlimit FROM pg_app_roles 
WHERE 1=1
AND rolname='usrqtoapp'
;

/*
GRANT ALL PRIVILEGES ON DATABASE dev_qto TO usrqtoapp ;
GRANT SELECT, INSERT, UPDATE, DELETE , TRIGGER ON ALL TABLES IN SCHEMA public TO usrqtoapp ;
GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO usrqtoapp;
GRANT SELECT ON TABLE pg_attribute TO usrqtoapp ;
GRANT SELECT ON TABLE pg_class TO usrqtoapp;
GRANT SELECT ON TABLE pg_index TO usrqtoapp;
GRANT SELECT ON TABLE pg_description TO usrqtoapp ;
GRANT SELECT ON TABLE pg_attrdef TO usrqtoapp ;

*/
<file_sep>#!/usr/bin/env bash


#v0.2.0
#------------------------------------------------------------------------------
# removes all the files from a deployed qto instance
#------------------------------------------------------------------------------
doTestRemovePackage(){

	doRemovePackageFiles
	
   #remove the dirs as well
   for dir in `cat "$include_file"`; do (
       dir="$PRODUCT_DIR/$dir"
       test -d "$dir" && cmd="rm -fRv $dir" && doRunCmdAndLog "$cmd"
   );
   done

 	cmd="rm -fv $include_file" && \
 	doRunCmdAndLog "$cmd"
	echo "rm -fvr $PRODUCT_DIR">>"$product_dir/remove-""$environment_name".sh
	echo "rm -fv $product_dir/remove-""$environment_name".sh>>"$product_dir/remove-""$environment_name".sh
	nohup bash "$product_dir/remove-""$environment_name".sh &
}
#eof test doRemovePackage
<file_sep>#!/bin/bash

do_setup_db_roles(){
   test -z "${PROJ_INSTANCE_DIR-}" && export PROJ_INSTANCE_DIR="$PRODUCT_DIR"
   test -z ${PROJ_CONF_FILE:-} && export PROJ_CONF_FILE="$PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json"
   do_export_json_section_vars $PROJ_CONF_FILE '.env.db'
   do_log "INFO using PROJ_INSTANCE_DIR: $PROJ_INSTANCE_DIR" ;
   do_log "INFO using PROJ_CONF_FILE: $PROJ_CONF_FILE"
   echo 'export PS1="`date "+%F %T"` \u@\h  \w \\n\\n  "' | sudo tee -a /var/lib/postgresql/.bashrc
   sudo -u root echo "postgres:$postgres_rdbms_usr_pw" | sudo chpasswd

   expect <<- EOF_EXPECT
      set timeout -1
      spawn sudo -u postgres psql --port $postgres_rdbms_port -c "\\\password"
      expect "Enter new password: "
      send -- "$postgres_usr_pw\r"
      expect "Enter it again: "
      send -- "$postgres_usr_pw\r"
      expect eof
EOF_EXPECT


   cd /tmp ; sudo -u postgres PGPASSWORD=$postgres_<PASSWORD> psql --port $postgres_rdbms_port -d postgres --host $postgres_rdbms_host -c "
	DO \$\$DECLARE r record;
		BEGIN
			IF NOT EXISTS (
				SELECT
				FROM   pg_catalog.pg_roles
				WHERE  rolname = '"$postgres_sys_usr_admin"') THEN
					CREATE ROLE "$postgres_sys_usr_admin" WITH SUPERUSER CREATEROLE
               CREATEDB REPLICATION BYPASSRLS PASSWORD '"$postgres_<PASSWORD>usr_<PASSWORD>"' LOGIN ;
			END IF;
		END\$\$;
	ALTER ROLE "$postgres_sys_usr_admin" WITH SUPERUSER CREATEROLE
   CREATEDB REPLICATION BYPASSRLS PASSWORD  '"$postgres_sys_usr_admin_pw"' LOGIN ; "
   sudo -u postgres PGPASSWORD=$postgres_usr_pw psql --port $postgres_rdbms_port --host $postgres_rdbms_host -c "
      grant all privileges on database postgres to $postgres_sys_usr_admin" ; cd -

}
<file_sep>#!/usr/bin/env bash

run_unit_bash_dir=$(perl -e 'use File::Basename; use Cwd "abs_path"; print dirname(abs_path(@ARGV[0]));' -- "$0")

# -- start add the qto admin user
sudo groupadd -g "10001" "grpqtoadmin"
sudo cat /etc/group | grep --color "grpqtoadmin"

sudo useradd --uid "10001" --home-dir "/home/usrqtoadmin" --gid "10001" \
   --create-home --shell /bin/bash "usrqtoadmin"
echo "usrqtoadmin:usrqtoadmin" | chpasswd

# obs !! dummy password <PASSWORD>
sudo cat /etc/passwd | grep --color "usrqtoadmin"
echo "usrqtoadmin ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
echo 'export PS1="`date "+%F %T"` \u@\h  \w \\n\\n  "' >> /home/usrqtoadmin/.bashrc
cat lib/bash/funcs/parse-cnf-env-vars.sh >> /home/usrqtoadmin/.bashrc

cat $(find cnf -name .vimrc|head -n 1) > /home/usrqtoadmin/.vimrc
mkdir -p /home/usrqtoadmin/opt

# -- start add the qto app user 
sudo groupadd -g "10002" "grpqtoapp"
sudo cat /etc/group | grep --color "grpqtoapp"

sudo useradd --uid "10002" --home-dir "/home/usrqtoapp" --gid "10002" \
   --create-home --shell /bin/bash "usrqtoapp"
echo "usrqtoapp:usrqtoapp" | chpasswd
echo 'export PS1="`date "+%F %T"` \u@\h  \w \\n\\n  "' >> /home/usrqtoapp/.bashrc
cat lib/bash/funcs/parse-cnf-env-vars.sh >> /home/usrqtoapp/.bashrc
cat $(find cnf -name .vimrc | head -n 1) > /home/usrqtoapp/.vimrc
<file_sep># ---------------------------------------------------------
# create a new <<env-type>> from this product instance
# bash src/bash/qto/qto.sh -a cp-to-env=dev -f met/.dev.qto
# bash src/bash/qto/qto.sh -a cp-to-env=tst -f src/bash/qto/qto.sh
# bash src/bash/qto/qto.sh -a cp-to-env=prd -f met/.dev.qto
# ---------------------------------------------------------
doCpToEnv(){
   tgt_env="${1:-}"
   prefix='cp-to-env='
   tgt_env=${tgt_env#$prefix}

	tgt_environment_name=$(echo $environment_name | perl -ne "s/$ENV/$tgt_env/g;print")
	tgt_PRODUCT_DIR=$product_dir/$tgt_environment_name

   # do ALWAYS open the met/<<current-env>> when calling this func !!!
   for env in `echo dev tst prd`; do cp -v met/.$ENV.qto met/.$env.qto ; done ;
   for env in `echo dev tst prd`; do cp -v met/.$ENV.qto $tgt_PRODUCT_DIR/met/.$env.qto ; done ;
	test "$tgt_env" == "$ENV" && return

   relative_fpath=$(dirname "${file}")
   tgt_file=$tgt_PRODUCT_DIR/$file
   
   mkdir -p $(dirname $tgt_file)
   test -f $file && cp -v $file $tgt_file
   test -d $tgt_file && rm -rv $tgt_file
   test -d $file && rsync -v -X -r -E -o -g --perms --acls $file $(dirname $tgt_file)

   # copy the current site instance configuration file to the secret store as well ...
   mkdir -p ~/.qto/cnf ; cp -v cnf/env/$ENV.env.json ~/.qto/cnf/
}
<file_sep># grab the list of cols straight from bash
export src_db='dev_qto'
export tgt_db='prd_phz'
export table_to_copy='app_item_attributes'
 psql -d "$src_db" -t -c \
 "SELECT column_name 
 FROM information_schema.columns 
 WHERE 1=1 
 AND table_name='"$table_to_copy"'"
 # ^^^ filter autogenerated cols if needed     

psql -h $postgres_rdbms_host -p $postgres_rdbms_port -d "$src_db" -c  \
 "copy ( SELECT guid, id, prio, table_name, name, description, skip_in_list, width, readonly, update_time FROM table_to_copy) TO STDOUT" |\
psql -d "$tgt_db" -c "copy $table_to_copy (guid, id, prio, table_name, name, description, skip_in_list, width, readonly, update_time) FROM STDIN"
<file_sep>CREATE OR REPLACE FUNCTION fnc_concat_ws(text, VARIADIC text[])
	RETURNS text LANGUAGE sql IMMUTABLE AS 'SELECT array_to_string($2, $1)';

SELECT 'fnc_concat_ws exists is ' || exists(SELECT * FROM pg_proc WHERE proname = 'fnc_concat_ws')<file_sep>place here any libraries you want to distribute with your qto ...
<file_sep>-- DROP TABLE IF EXISTS test_hi_delete_table_doc CASCADE;

SELECT 'delete the "test_hi_delete_table_doc" table' as "---"
;

-- src: http://mikehillyer.com/articles/managing-hierarchical-data-in-mysql/
-- src: https://github.com/werc/TreeTraversal
   CREATE TABLE test_hi_delete_table_doc (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISSMS') as bigint) 
    , level          integer NOT NULL DEFAULT 1
    , seq            integer NOT NULL DEFAULT 1
    , lft            integer NOT NULL DEFAULT 1
    , rgt            integer NOT NULL DEFAULT 2
    , name           varchar (200) NOT NULL DEFAULT 'name...'
    , description    varchar (4000) NULL
    , formats        varchar (200) NULL 
    , src            varchar (4000) NULL
    , CONSTRAINT pk_test_hi_delete_table_doc_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

-- STOP  delete table test_hi_delete_table_doc
-- --------------------------------------------------------

-- INSERT INTO test_hi_delete_table_doc ( id,level,seq,lft,rgt,name) 
  --                      values ( 0,0,1,1,2,'test_hi_delete_table_doc');


SELECT 'Display the columns of the just deleted table' as "---"
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.test_hi_delete_table_doc'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 


<file_sep># file: src/bash/qto/funcs/generate-pdf-docs.func.sh

doGeneratePdfDocs(){
   test -z "${PROJ_INSTANCE_DIR-}" && PROJ_INSTANCE_DIR="$PRODUCT_DIR"
   test -z "${docs_root_dir-}" && docs_root_dir="$PROJ_INSTANCE_DIR"

   test -z ${PROJ_CONF_FILE-} && PROJ_CONF_FILE=$PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json
   do_export_json_section_vars $PROJ_CONF_FILE '.env.app'

   # <<web-host>>:<<web-port>>/<<db>>/select/export_files?as=grid&od=id
   basic_url="$ht_protocol://${web_host:-}:${port:-}/${postgres_app_db:-}"
   furl="$basic_url"'/select/export_files?as=grid&od=id&pg-size=20'
   echo "running: curl --cookie ~/.qto/cookies.txt --insecure  -s $furl \| jq -r '.dat[]|.url'"
   curl --cookie ~/.qto/cookies.txt --insecure  -s $furl | jq -r '.dat[]|.url'
	ret=$?
	test $ret != "0" && do_exit $ret "failed to get data from the $furl"

   while read -r url ; do 
      file_name=$(curl --cookie ~/.qto/cookies.txt --insecure  -s $furl | jq -r '.dat[]| select(.url=='\"$url\"')| .name'); 
      file_name="$file_name.pdf"
      rel_path=$(curl --cookie ~/.qto/cookies.txt --insecure  -s $furl | jq -r '.dat[]| select(.url=='\"$url\"')| .path');
      [ "${rel_path-}" == "null" ] && rel_path="doc"
      rel_path="$rel_path/pdf"
      mkdir -p $docs_root_dir/$rel_path
      file_path=$(echo $docs_root_dir/$rel_path/$file_name|perl -ne 's|[/]{2,5}|/|g;print')
      url="$url"'?as=print-doc'
      echo -e "\nrunning: chrome --headless --disable-gpu --print-to-pdf=$file_path $basic_url/view/$url"
      chrome --headless --disable-gpu --print-to-pdf=$file_path $basic_url/view/$url
      lines=$(wc -l "$file_path"|awk '{print $1;}')
      [[ $lines -eq 0 ]] && do_exit 0 "load the $url !!! the table is empty !!!"
      echo -e "$lines lines in the $file_path file \n"
   done < <(curl --cookie ~/.qto/cookies.txt --insecure  -s $furl | jq -r '.dat[]|.url')
	
   do_flush_screen
}

<file_sep># v1.2.9 
#------------------------------------------------------------------------------
# scan all the "defined" actions and generate the 
# spec , func , test , doc files if the do not exist 
#------------------------------------------------------------------------------
doGenerateActionFiles(){
	
	do_log "DEBUG START : doGenerateActionFiles"
	
	# for each defined action 	
	while read -r act ; do (

		do_log "DEBUG START :: checking action: $act"
			# for each defined action 	
			while read -d" " deliverable_type ; do (

				do_log "DEBUG START ::: action: $act - deliverable_type: $deliverable_type "
				# src: https://gist.github.com/tyru/358703
				func_part_name=$(echo $act|perl -ne 's{(\w+)}{($a=lc $1)=~s<(^[a-z]|-[a-z])><($b=uc$1);$b;>eg;$a;}eg;s|-||g;print')
				do_log "DEBUG func_part_name: $func_part_name"
				
				# foreach deliverable_type build the code and doc files	
				case $deliverable_type in 
					'spec')
					#
						full_func="doSpec""$func_part_name"
						deliverable_doc_file="doc/txt/$RUN_UNIT/specs/$act.spec.txt"
						deliverable_code_file="src/bash/$RUN_UNIT/specs/$act.spec.sh"
					;;
					'func')
						full_func="do""$func_part_name"
						deliverable_doc_file="doc/txt/$RUN_UNIT/funcs/$act.func.txt"
						deliverable_code_file="src/bash/$RUN_UNIT/funcs/$act.func.sh"
					;;
					'test')
						full_func="doTest""$func_part_name"
						deliverable_doc_file="doc/txt/$RUN_UNIT/tests/$act.test.txt"
						deliverable_code_file="src/bash/$RUN_UNIT/tests/$act.test.sh"
					;;
					'help')
						full_func="doHelp""$func_part_name"
						deliverable_doc_file="doc/txt/$RUN_UNIT/helps/$act.help.txt"
						deliverable_code_file="src/bash/$RUN_UNIT/helps/$act.help.sh"
					;;
				esac

				do_log " DEBUG full_func: $full_func"
				do_log " INFO delvr_doc_file: $deliverable_doc_file"
				do_log " INFO delvr_code_file: $deliverable_code_file"
				do_log " DEBUG STOP  ::: action: $act - deliverable_type: $deliverable_type "

				# if the delivable file does not exist create it
				code_file_exists=$(find "src/bash/$RUN_UNIT/$deliverable_type""s" | grep $act.$deliverable_type.sh| wc -l)
				if [ $code_file_exists -eq 0 ];then

					cp -v src/bash/$RUN_UNIT/funcs/%act%.%deliverable_type%.sh "$deliverable_code_file"
					perl -pi -e "s|%full_func%|$full_func|g" "$deliverable_code_file"
					perl -pi -e "s|%act%|$act|g" "$deliverable_code_file"
					perl -pi -e "s|%deliverable_type%|$deliverable_type|g" "$deliverable_code_file"

				fi
				
				doc_file_exists=$(find "doc/txt/$RUN_UNIT/$deliverable_type""s" | grep $act.$deliverable_type.txt| wc -l)
				if [ $doc_file_exists -eq 0 ];then

					cp -v doc/txt/$RUN_UNIT/tmpl/%act%.%deliverable_type%.txt "$deliverable_doc_file"
					perl -pi -e "s|%full_func%|$full_func|g" "$deliverable_doc_file"
					perl -pi -e "s|%act%|$act|g" "$deliverable_doc_file"
					perl -pi -e "s|%deliverable_type%|$deliverable_type|g" "$deliverable_doc_file"

				fi
			); 
			done< <(echo 'spec' 'func' 'test' 'help' 'none')

      echo -e "generated the following files: \n" ; 
      find . | grep -i $act |cut -c 3-|sort -nr
      find . | grep -i $act |cut -c 3-|sort -nr >> met/.$ENV.$RUN_UNIT
      echo -e "\n\n" 

		do_log "DEBUG STOP  :: checking action: $act"
		
	); 
	done< <(cat "src/bash/$RUN_UNIT/tests/new-$RUN_UNIT-tests.lst")
   clear ; for env in `echo dev tst prd src`; do cp -v met/.$ENV.$RUN_UNIT met/.$env.$RUN_UNIT ; done
	
	do_log "DEBUG STOP  : doGenerateActionFiles"

}
#eof func doGenerateActionFiles
<file_sep># src/bash/qto/funcs/print-help.help.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doHelpPrintHelp comments ...
# ---------------------------------------------------------
doHelpPrintHelp(){

	do_log "DEBUG START doHelpPrintHelp"
	
	cat doc/txt/qto/helps/print-help.help.txt
	sleep 2
	# add your action implementation code here ... 

	do_log "DEBUG STOP  doHelpPrintHelp"
}
# eof func doHelpPrintHelp


# eof file: src/bash/qto/funcs/print-help.help.sh
<file_sep>#  QTO DEVOPS GUIDE
* [1. INTRODUCTION](#1-introduction)
* [2. GUIDING PRINCIPLE'S](#2-guiding-principle's)
  * [2.1. USE COMMON SENSE](#21-use-common-sense)
  * [2.2. PERSONAL RESPONSIBILITY](#22-personal-responsibility)
  * [2.3. ATTEMPT FOR 100% TEST COVERAGE TO ACHIEVE RELIABILITY](#23-attempt-for-100%-test-coverage-to-achieve-reliability)
  * [2.4. IT SHOULD JUST WORK](#24-it-should-just-work)
  * [2.5. NAMING CONVENTIONS](#25-naming-conventions)
  * [2.6. BE FRIENDLY TO ALL](#26-be-friendly-to-all)
  * [2.7. AIM FOR SIMPLICITY](#27-aim-for-simplicity)
  * [2.8. DO NOT ALLOW BROKEN WINDOWS](#28-do-not-allow-broken-windows)
  * [2.9. DO NOT ADD A COMMIT WITHOUT PASSING ALL THE REGRESSION TESTS](#29-do-not-add-a-commit-without-passing-all-the-regression-tests)
* [3. WAY OF WORKING AND PROCESSES IN THE QTO PROGRAM](#3-way-of-working-and-processes-in-the-qto-program)
  * [3.1. VERBAL COMMUNICATION, PHONE & VIDEO CONFERENCING](#31-verbal-communication-phone-&-video-conferencing)
  * [3.2. CHAT / IRC](#32-chat-/-irc)
  * [3.3. DEFINITION OF DONE](#33-definition-of-done)
  * [3.4. E-MAIL COMMUNICATION](#34-e-mail-communication)
  * [3.5. CONTINUOUS IMPROVEMENT BY ISSUES MANAGEMENT](#35-continuous-improvement-by-issues-management)
* [4. ETL OPERATIONS](#4-etl-operations)
  * [4.1. DB DATA BACKUP AND RESTORE](#41-db-data-backup-and-restore)
  * [4.2. RUN INCREASE-DATE ACTION](#42-run-increase-date-action)
  * [4.3. LOAD XLS ISSUES TO DB AND FROM DB](#43-load-xls-issues-to-db-and-from-db)
  * [4.4. DIRS NAMING CONVENTIONS](#44-dirs-naming-conventions)
* [5. NAMING CONVENTIONS](#5-naming-conventions)
  * [5.1. PRODUCT INSTANCE DIRECTORIES](#51-product-instance-directories)
  * [5.2. ROOT DIRS NAMING CONVENTIONS](#52-root-dirs-naming-conventions)
  * [5.3. DABASE NAMING CONVENTIONS](#53-dabase-naming-conventions)
* [6. SOURCE CODE MANAGEMENT](#6-source-code-management)
  * [6.1. CONFIGURE AND USE GIT ALWAYS BY USING SSH IDENTITIES](#61-configure-and-use-git-always-by-using-ssh-identities)
  * [6.2. AIM FOR TRACEABILITY BETWEEN USER-STORIES, REQUIREMENTS, FEATURES AND FUNCTIONALITIES](#62-aim-for-traceability-between-user-stories-requirements-features-and-functionalities)
  * [6.3. ZERO TOLERANCE FOR BUGS, ESPECIALLY CRASHES](#63-zero-tolerance-for-bugs-especially-crashes)
  * [6.4. ALWAYS START WITH A TEST UNIT CREATION](#64-always-start-with-a-test-unit-creation)
  * [6.5. BRANCH FOR THE CURRENT COMMON RELEASABLE BRANCH - FOR EXAMPLE V0.8.4](#65-branch-for-the-current-common-releasable-branch--for-example-v084)
  * [6.6. SYNCING CHANGES WITH THE ORIGINAL REPO](#66-syncing-changes-with-the-original-repo)
  * [6.7. INTEGRATION TESTING IN THE TST BRANCH](#67-integration-testing-in-the-tst-branch)
  * [6.8. PRODUCTION IN THE PRD BRANCH](#68-production-in-the-prd-branch)
  * [6.9. DEPLOYMENT TO THE CLOUD](#69-deployment-to-the-cloud)
  * [6.10. MASTER BRANCH - THE SINGLE TRUTH FOR CURRENT STABLE VERSION OF THE SOFTWARE](#610-master-branch--the-single-truth-for-current-stable-version-of-the-software)
* [7. SOURCE CODE STYLING](#7-source-code-styling)
  * [7.1. SPACES AND NOT TABS](#71-spaces-and-not-tabs)
  * [7.2. PERL SOURCE CODE STYLING](#72-perl-source-code-styling)
    * [7.2.1. Spacing after variable asignments](#721-spacing-after-variable-asignments)
    * [7.2.2. Set 3 spaces for a tab](#722-set-3-spaces-for-a-tab)
    * [7.2.3. Empty lines](#723-empty-lines)
* [8. TESTING](#8-testing)
  * [8.1. RUNNING UNIT TESTS](#81-running-unit-tests)
  * [8.2. CHECKING THE PERL SYNTAX](#82-checking-the-perl-syntax)
  * [8.3. RUNNING FUNCTIONAL TESTS](#83-running-functional-tests)
  * [8.4. RUNNING INTEGRATION TESTS](#84-running-integration-tests)
  * [8.5. USER-STORY CREATION](#85-user-story-creation)
* [9. FEATURE IMPLEMENTATION WORKFLOW](#9-feature-implementation-workflow)
  * [9.1. PROBLEM REGISTRATION](#91-problem-registration)
  * [9.2. REQUIREMENTS CREATION](#92-requirements-creation)
  * [9.3. ISSUE CREATION](#93-issue-creation)
  * [9.4. FEATURE BRANCH CREATION](#94-feature-branch-creation)
  * [9.5. CREATE A TEST-ENTRY POINT](#95-create-a-test-entry-point)
  * [9.6. IMPLEMENTATION OF PROOF OF CONCEPT ( OPTIONAL )](#96-implementation-of-proof-of-concept-(-optional-))
  * [9.7. PROTOTYPE IMPLEMENTATION ( OPTIONAL )](#97-prototype-implementation-(-optional-))
  * [9.8. UNIT AND / OR INTEGRATION TEST CREATION](#98-unit-and-/-or-integration-test-creation)
  * [9.9. IMPLEMENTATION ](#99-implementation-)
  * [9.10. DEPLOYMENT AND TEST TO THE TEST ENVIRONMENT](#910-deployment-and-test-to-the-test-environment)
  * [9.11. DEPLOYMENT AND TEST TO THE PRODUCTION ENVIRONMENT](#911-deployment-and-test-to-the-production-environment)
  * [9.12. QUALITY ASSURANCE ITERATION](#912-quality-assurance-iteration)
  * [9.13. DOD CHECK-LIST WALKTHROUGH](#913-dod-check-list-walkthrough)
  * [9.14. THE FEATURE OR FUNCTIONALITY CURRENT DESCRIPTION IS ADDED IN THE DOCS](#914-the-feature-or-functionality-current-description-is-added-in-the-docs)
    * [9.14.1. Regenerate the md docs](#9141-regenerate-the-md-docs)
    * [9.14.2. Regenerate the pdf docs](#9142-regenerate-the-pdf-docs)
    * [9.14.3. Regenerate the msft docs](#9143-regenerate-the-msft-docs)
  * [9.15. THE RELATED REQUIREMENT IS ADDED IN THE REQUIREMENTS DOCUMENT](#915-the-related-requirement-is-added-in-the-requirements-document)
  * [9.16. AT LEAST 2 TIMES PASSED FUNCTIONAL AND JS TESTS RUN ](#916-at-least-2-times-passed-functional-and-js-tests-run-)
  * [9.17. AT LEAST 2 TIMES PASSED INTEGRATION TESTS  IN EACH ENVIRONMENT INSTANCE](#917-at-least-2-times-passed-integration-tests-in-each-environment-instance)
  * [9.18. DEPLOYMENT TO THE TEST ENVIRONMENT](#918-deployment-to-the-test-environment)
  * [9.19. CHECK THAT ALL THE FILES IN THE DEPLOYMENT PACKAGE ARE THE SAME AS THOSE IN THE LATEST COMMIT OF THE DEV GIT BRANCH. ](#919-check-that-all-the-files-in-the-deployment-package-are-the-same-as-those-in-the-latest-commit-of-the-dev-git-branch-)
  * [9.20. RESTART THE APPLICATION LAYER](#920-restart-the-application-layer)
* [10. DEVBOX SETUP ( OPTIONAL )](#10-devbox-setup-(-optional-))
  * [10.1. THE TMUX TERMINAL MULTIPLEXER](#101-the-tmux-terminal-multiplexer)
  * [10.2. THE VIM IDE](#102-the-vim-ide)
* [11. DEBUGGING](#11-debugging)
  * [11.1. ENABLE DEBUGGING IN MOJOLICIOUS](#111-enable-debugging-in-mojolicious)
  * [11.2. ENABLE DEBUGGING IN DBI](#112-enable-debugging-in-dbi)
  * [11.3. DEBUGGING IN THE UI](#113-debugging-in-the-ui)
  * [11.4. DEBUGGING CSS](#114-debugging-css)
* [12. SECURITY](#12-security)
  * [12.1. AUTHENTICATION](#121-authentication)
    * [12.1.1. JWT based native authentication](#1211-jwt-based-native-authentication)
    * [12.1.2. Session based native authentication](#1212-session-based-native-authentication)
    * [12.1.3. RBAC based native authentication](#1213-rbac-based-native-authentication)
  * [12.2. AUTHORISATION](#122-authorisation)
    * [12.2.1. Generic role-based access control list based authorisation](#1221-generic-role-based-access-control-list-based-authorisation)
* [13. KNOWN ISSUES AND WORKAROUNDS](#13-known-issues-and-workarounds)
  * [13.1. THE RBAC BASED NATIVE AUTHORISATION](#131-the-rbac-based-native-authorisation)
  * [13.2. ALL TESTS FAIL WITH THE 302 ERROR](#132-all-tests-fail-with-the-302-error)
  * [13.3. MORBO IS STUCK](#133-morbo-is-stuck)
    * [13.3.1. Probable root cause](#1331-probable-root-cause)
    * [13.3.2. Kill processes](#1332-kill-processes)
    * [13.3.3. Problem description](#1333-problem-description)
  * [13.4. THE PAGE LOOKS BROKEN - PROBABLY THE NEW CSS IS NOT RE-LOADED](#134-the-page-looks-broken--probably-the-new-css-is-not-reloaded)
  * [13.5. THE VUE UI DOES NOT UPDATE PROPERLY ](#135-the-vue-ui-does-not-update-properly-)
  * [13.6. NGINX FAILS WITH 502 BAD GATEWAY ERROR AND PROBABLY CRASHES THE SITE](#136-nginx-fails-with-502-bad-gateway-error-and-probably-crashes-the-site)
  * [13.7. WHY HAVING ALL THE HASSLE WITH THIS DIRECTORY STRUCTURE - IS OVERKILL ?!!](#137-why-having-all-the-hassle-with-this-directory-structure--is-overkill-)
  * [13.8. THE NEW ENTRIES IN DROPBOX FK TABLES CANNOT BE SEEN](#138-the-new-entries-in-dropbox-fk-tables-cannot-be-seen)
* [14. FAQ](#14-faq)




    

## 1. INTRODUCTION


    

## 2. GUIDING PRINCIPLE'S
This section might seem too philosophical for a start, yet all the development in the qto has ATTEMPTED to follow the principles described below. If you skip this section now you might later on wander many times why something works and it is implemented as it is ... and not "the right way". 
Of course you are free to not follow these principles, the less you follow them the smaller the possibility to pull features from your instance(s) - you could even use the existing functionality to create a totally different fork with different name and start developing your own toll with name X - the authors give you the means to do that with this tool ... , but if you want to use and contribute to THIS tool than you better help defined those leading principles and follow them. 

    

### 2.1. Use common sense
Use common sense when applying all those principles. Of course they are not engraved in stone and you should be flexible enough for the actual situation, problem, issue etc.

    

### 2.2. Personal responsibility
Any given instance of the qto should have ONE and only ONE person which is responsible at the end for the functioning of THE instance - so think carefully before attempting to take ownership of an instance. The author(s) of the code are not responsible for the operation, bugs or whatever happens to a new instance. As a responsible owner of an instance you could create, share and assign issues to the authors of the source code, yet there is no Service Level Agreement, only openly stated attempt to assist when possible.
Qto is design to make version updates fully vertically integrated and as automated as possible, yet YOU will be responsible for increasing the versions, taking backups, applying database migrations and so on ... 

    

### 2.3. Attempt for 100% test coverage to achieve reliability
The more you increase your test coverage the greater the confidence that the code will work as expected. 
Do not write a single function without first implementing the testing call for that function - this has been proven really, really difficult, yet the more features are added the less the time wasted in troubleshooting of bugs and unexpected behaviour when proper testing is implemented. 
Testing ensures the consistency and future expandability of the functionalities. Our velocity increases with the WORKING features and functionalities added over time. ANYTHING, which is not working, or not even sure about how it should be working MUST be [fixed](fixed/deleted.) 

    

### 2.4. It should just work
Any instance of the QTO should simply work as promised - no less, no more. 
Any instance is the combination of code, configurations, binaries in the System and data - that is the instance you are using should just work for the set of functionalities promised. 

    

### 2.5. Naming conventions
All the names used in the code and the configurations MUST BE human readable and expandable - that is name the objects from the greater realm to the smaller - for example &lt;&lt;env&gt;&gt;_&lt;&lt;db_name&gt;&gt; , because the concept of operational IT environments ( dev , test , qas , prd ) is broader than the concept of a application databases . 
Before you start a new concept for example code for a new run-time spent some time first for some initial planning:
 - how many objects are there going to be after 10 years !!
 - how about searching through those objects - how easy will it be 
 - are you naming data, configuration, binary or source code objects ?

    

### 2.6. Be friendly to all
Especially to technical personnel, as you cannot achieve user-friendliness for the end-users unless your developers and technical personnel are happy while interacting with your artefacts. 

    

### 2.7. Aim for simplicity
Things should be as simple as possible, but not simpler - if Einstein said it it makes sense - having lost so much time in endless loops of IT complexity - the older we get the more it gets more rational. 

    

### 2.8. Do not allow broken windows
A broken windows is any peace of code or documentation, which is hanging around not included in the integration tests suite and not matching the most up-to-date standards for work deliverables. Either bring it up to the standard level or get rid of it. 
As soon as you find a bug, write a test for it, if you can't create the needed testing setup invest in time developing the needed skills.

    

### 2.9. Do not add a commit without passing ALL the regression tests
Even in your personal branch. Really. Because after the application has surpassed the mark of having 200 000 lines of code the complexity added to a "broken machine" WILL NOT justify the breaking of an existing feature. If you do not consider the feature / functionality tested as important than feel free to REMOVE it ( both implementations AND tests ) in that very same commit.

    

## 3. WAY OF WORKING AND PROCESSES IN THE QTO PROGRAM
A project represents a single, focused endeavour, whereas a program might be even a collection of projects, with ongoing activities and not a clear, definite, SINGLE outcome. 
Qto is more of a program with not strictly defined end-product and less of a project, so whenever you see the "qto project" think more for a "definite milestone of the qto program".
This section describes the way of working within a team working on the qto program. 
The work on the qto project is conducted by using the Scrum methodology, thus the Scrum

    

### 3.1. Verbal communication, phone & video conferencing
Time is money. Let's assume that a meeting of 7 IT Consultants charging 100 $ per hour should have meeting for 1.5 hours - that is an investment from the paying organisation point of view for 1050 $. Let's assume that 2 guys are late 5min, 1 guy 10min and during that time 1 guy tells about the wonderful trip he had during the weekend and the latest sport events ... so the meetings ACTUALLY starts after the first 20 min ( about 1/3 of the investment is lost already).  

    

### 3.2. Chat / IRC
Should you want a quicker respond than 2 hours use a chat tool. Do not expect people to answer you straight away, it takes 5 to 20 min to reach the most productive flow state, thus not answering your question might be the more productive option from the point of view of the organisation.

    

### 3.3. Definition of Done
Each issue must have a tangible artifact. An issue without tangible artifact is a thought thrown in the air.
The DoD must be iterated and updated during each Sprint Review. 

    

### 3.4. E-mail communication
Do not use e-mail communication for code style, testing, developing etc. Issues which could be achieved with the code review interface of the source code management system. 
Before writing an e-mail think first could you find a way to avoid writing it at all. 
Do not expect answer of your e-mail within 2 hours. 
Use e-mail when you have to get an written evidence on agreed matters, which might cause later on discussions. 

    

### 3.5. Continuous improvement by issues management
At the end of the month you should move the completed issues to the yearly_issues table as follows:


    

## 4. ETL OPERATIONS


    

### 4.1. Db data backup and restore
Check maintenance_guide_doc-23

    

### 4.2. Run increase-date action
You track the issues of your projects by storing them into xls files in "daily" proj_txt dirs. 
Each time the day changes by running the increase-date action you will be able to clone the data of the previous date and start working on the current date.

    bash src/bash/qto/qto.sh -a increase-date

### 4.3. Load xls issues to db and from db
To load xls issues to db run the following one-liner:

    bash src/bash/qto/qto.sh -a xls-to-db
    
    

### 4.4. Dirs naming conventions
The dir structure should be logical and a person navigating to a dir should almost understand what is to be find in there by its name. 

    

## 5. NAMING CONVENTIONS


    

### 5.1. Product instance directories


    

### 5.2. Root Dirs naming conventions
The root dirs and named as follows:
bin - contains the produced binaries for the project
cnf - for the configuration
dat - for the data of the app
lib - for any external libraries used
src - for the source code of the actual projects and subprojects

    

### 5.3. Dabase naming conventions
Each database must start with its environment prefix - dev, tst or prd. And yes this is so fundamentally in-built into QTO that changing this naming convention will definitely destroy your application.

    

## 6. SOURCE CODE MANAGEMENT
The QTO is a derivative of the wrapp tool - this means that development and deployment process must be integrated into a single pipeline. 

    

### 6.1. Configure and use git ALWAYS by using ssh identities
You probably have access to different corporate and public git repositories. Use your personal ssh identity file you use in GitHub to push to the QTO project. The following code snippet demonstrates how you could preserve your existing git configurations (even on corporate / intra boxes), but use ALWAYS the personal identity to push to the QTO.

    # create the company identity file
    ssh-keygen -t rsa -b 4096 -C "<EMAIL>" -f ~/.ssh/id_rsa.corp.`hostname -s`
    
    ~/.ssh/id_rsa.corp.`hostname -s` 
    # copy paste this one into your githubs, private keys
    
    # set alias for the git command to avoid overtyping ...
    alias git='GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa.corp.`hostname -s` " git'
    
    # clone a repo 
    git clone <EMAIL>:corp/project.git
    
    export git_msg="my commit msg with my corporate identity, explicitly provide author"
    git add --all ; git commit -m "$git_msg" --author "MeFirst MeLast <<EMAIL>>"
    git push 
    
    # and verify 
    clear ; git log --pretty --format='%h %ae %<(15)%an ::: %s

### 6.2. Aim for traceability between user-stories, requirements, features and functionalities
Once the issues are defined and you start working on your own branch which is named by the issue-id aim to map one on one each test in your code with each listed requirement in confluence and / or JIRA. 

    

### 6.3. Zero tolerance for bugs, especially crashes
As soon as bugs are identified and reproduceable, register them as issues and resolve them with prio ( aka priority ) 1.
After resolution, think about the root cause of the bug, the mear fact that the bug occurred tells that something in the way of working has to be improved , what ?!
Bugs are like broken windows the more you have them the faster the value of your building will be down to zero. 

    

### 6.4. ALWAYS Start with a test unit creation
Do not ever never write code without starting first the unit test on how-to test the code. Period. 
This is he only way to avoid braking old functionalities when the application code base grows larger. 
Each time a new bug is found fix it by adding new Unit Test!

    

### 6.5. Branch for the current common releasable branch - for example v0.8.4
No code should be merged into the development branch without broad testing coverage and approval from the owner of the instance - as the owner of the instance is at the end responsible personally for the whole instance, since once a change has been merged to develop it must pass as quickly as possible to tst, prd and master.
Each developer committing to the development branch MUST rebase as frequently as possible from the dev branch to avoid time waste aligning the feature branches to the common dev branch.

    # put your product instance configuration files into the stash temporarily
    git stash 
    
    # check the current wip version of the qto application
    git checkout v0.8.4
    
    # create a new branch from the v0.8.3 branch AND set the correct name of the branch
    # by using the issues-id and use for the name of the branch the copy paste from the issue
    # name by replacing the spaces and other special chars with - dashes ... 
    git checkout -b v0.8.4-200402120006-fix-bug-for-wrong-order-in-env-file
    
    # apply back your configuration
    git stash pop
    
    # modify the code and strife to modify code ONLY related to THIS issue !!!
    # set the git msg 
    export git_msg="qto-200402120006 fix bug for wrong order in .env file"
    
    # push the changes ( you would have to have the git alias set !!!)
    git add --all ; git commit -m "$git_msg" --author "<NAME> <<EMAIL>>" ; git push ;
    
    # have to set a new upstream when pushing for the first time 
    git push --set-upstream origin v0.8.4-fix-bug-for-wrong-order-in-env-file
    
    # set the 
    git stash
    git branch -a
    git checkout v0.8.4
    
    git pull
    
    git merge v0.8.3-200402120006-fix-bug-for-wrong-order-in-env-file -X theirs
    git push
    git stash pop
    git push 
    clear ; git log --pretty --format='%h %<(15)%ae %<(15)%an ::: %s'

### 6.6. Syncing changes with the original repo
If you have forked a GitHub repo you want to sync with you will need to  :
 - add it to your git remotes ( if it is not added yet ) 
 - add the upstream repository ( or update the url if was existing )
 - get the upstream changes to your local branch(es)
 - push to your remote remote branch(es)


    #use git remote -v to see the currently configured repositories you are working with:
    git remote -v
    
    #include your own repo:
    git remote set-url origin https://github.com/%YourName%/qto.git
    
    #add https://github.com/YordanGeorgiev/qto/ as an upstream repository:
    git remote add upstream https://github.com/YordanGeorgiev/qto/qto.git
    
    #in case you already have an upstream configured, you can update the URL instead of creating it:
    git remote set-url upstream https://github.com/YordanGeorgiev/qto.git
    
    #ensure that origin is set to your repo and upstream is set to the main QTO repo:
    git remote -v 
    
    #fetch all the changes from upstream to your local files:
    git fetch upstream
    
    #check that you are on the desired branch, usually it is master:
    git checkout master
    
    #merge the changes from the upstream/master into your local master branch. 
    #it is also possible to specify a different upstream branch other than master, for example upstream/v0.8.4:
    git merge upstream/master
    
    #push or force push to your Github master branch:
    git push origin master
    

### 6.7. Integration testing in the tst branch
The tst branch is dedicated for integration testing of all the tests, the deployment, performance testing and configuration changes. Should you need to perform bigger than a small hotfix changes you must branch the tst branch into a separate dev--feature branch and re-run the integration testing and approval all over. 
At the end all the integration tests should be behind this shell call. 

    bash src/bash/qto/qto.sh -a run-integration-tests

### 6.8. Production in the prd branch
The prd branch is the one deployed to the production environment. This code is NOT straight merged into the master branch , but after certain time depending on the dynamic of the tool with bugless operation merged. 

    

### 6.9. Deployment to the cloud
//

    

### 6.10. master branch - the single truth for current stable version of the software
Once the business has approved a new version - it should be moved to the master branch and all other branches including the separate feature branches MUST be REBASED ( and NOT MERGED !!! ) from the master branch to accommodate any hotfixes, configuration related adjustments or quick bug fixes detected in production only. 

    

## 7. SOURCE CODE STYLING
This section is going to be the one to be debated and hated the most ...

    

### 7.1. Spaces and not tabs


    

### 7.2. Perl source code styling


    

#### 7.2.1. Spacing after variable asignments
You might want to use spacing after variable assignments if you have a lot of variables assigned ... it improves the readability of the code ....

    

#### 7.2.2. Set 3 spaces for a tab
Why ????!!!!!
Because 2 is too little and 4 is too much and line legth of 80 to 100 is too small to be wasted or packed with too much stuff ... 
Also 3 is a magical number ...

    

#### 7.2.3. Empty lines
Empty lines improve readability, when used in a standard way: 
Use 2 empty lines between functions and 1 empty line between logical blocks within functions.

    

## 8. TESTING
Why the testing section is before the coding one ??!!!
Because in order for you to be able to add more functionalities to the qto application you MUST first understand how to ensure that you are not causing regression bugs - aka the application "works" according to one of the main principle - it must just work, no more and no less.

    

### 8.1. Running unit tests


     bash src/bash/qto/qto.sh -a run-unit-tests

### 8.2. Checking the perl syntax
Before running any tests check the perl syntax ... as follows:

    bash src/bash/qto/qto.sh -a check-perl-syntax
    # starts echoging something like this...
    ...
     ::: running: cd src/perl/qto ; perl -MCarp::Always -I /hos/opt/qto/qto.0.8.1.dev.ysg/src/perl/qto -I /hos/opt/qto/qto.0.8.1.dev.ysg/src/perl/qto/lib -wc "./t/lib/Qto/App/Utils/TestInitiator.t" ; cd -

### 8.3. Running functional tests


     bash src/bash/qto/qto.sh -a run-functional-tests

### 8.4. Running integration tests


     bash src/bash/qto/qto.sh -a run-integration-tests

### 8.5. User-Story creation
Use the following template while creating the user story:
As an &lt;&lt;role&gt;&gt;
In order to &lt;&lt;achieve something/ bring value by&gt;&gt; 
I want to to be able to &lt;&lt;action-description&gt;&gt; 

    

## 9. FEATURE IMPLEMENTATION WORKFLOW
This section describes the common workflow for implementing a feature. 
As in other places the main principle to follow is "use common sense" , thus try to follow this workflow for feature implementation, but challenge it as soon as it defies the common sense. 

    

### 9.1. Problem registration
Should you have problems or bugs, or even questions register them first into the qto application. Problems are often hint from the reality on the actual work, which has to be performed, yet problems are not yet issues. Issues are meant to be derived from problems in structured way, so that the deliverable of each issue must be testable and distinguishable, as well as the work.
The reason the problem registration is on top of this subsection is the fact that bugs and problems should be dealt with higher priority, to minimise the technical debt of the application.

    

### 9.2. Requirements creation
Depending on the size and agility of your organisation formal requirements exist.

    

### 9.3. Issue creation
Even if you do not have a defined documentation artefact - create a new issue, which could be the start for a an action affecting the run-state, configuration , data , features and functionalities or other aspects of the qto application. 
An issue could be a bug, a request for a feature or even simply an undefined combination of problems and solution which could quickly be formalised by defining a new requirement, another issue, feature-request

    

### 9.4. Feature branch creation
Create the feature branch by using the following naming convention:
 - dev--&lt;&lt;short-feature-title&gt;&gt;

    git branch -a
    * dev
    dev--qto-18050801-add-order-by-in-select-ctrlr
    master
    ...
    
    

### 9.5. Create a test-entry point
Even the smallest proof of concept needs a small test-entry point. Start always with the testing and the testing scalability in mind. 

    

### 9.6. Implementation of Proof of Concept ( optional )
Aim to create a small POC for the new concept, feature or functionality - for example a page having a lot of hardcoding, which constrains the scope for ONLY this new thing. 
POC small "projects" are extremely useful when new ui controls are to be integrated into the common base - check the public/poc directory for the the subdirectories containing the iterative approaches while introducing different controls into the project.
Strive however to use the same naming convention, and implement with future integrations within the end truly dynamic code. 

    

### 9.7. Prototype implementation ( optional )
The same instructions as the POC apply, but the prototype contains a certain and broader level of integration with the dynamic parts of the System.

    

### 9.8. Unit and / or integration test creation
Strive to create always unit and / or integration test(s).

    

### 9.9. Implementation 
Implement by quick unit test runs. Constantly improve both the code , configuration changes and the test code. Think about re-usability and scalability during implementation, but do not overgeneralise.

    

### 9.10. Deployment and test to the test environment
Deploy to the test environment a new instance with the current version as follows:

    # deploy to the tst environment
    bash src/bash/qto/qto.sh -a to-tst
    
    # go to the product instance dir of the tst env for this version
    cd ../qto.<<version>>.tst.<<owner>>
    bash src/bash/qto/qto.sh -a run-integration-tests
    bash src/bash/qto/qto.sh -a run-functional-tests

### 9.11. Deployment and test to the production environment
Repeat the same to the production environment. As the current version is usually work in progress your stable version will be one level below and thanks to the architecture of the tool you could test in the production environment ( as soon as you have proper configuration ).

    

### 9.12. Quality assurance iteration
This phase might be longer depending on the feature. Some of the features stay in quality assurance mode EVEN if they have been deployed to production.

    

### 9.13. DoD check-list walkthrough
Perform the DoD checklist as follows.

    

### 9.14. The feature or functionality current description is added in the docs
The feature or functionality current description is added in the Features and Functionalities document. Check also the end user guide - it might to be changed as well. It might pay off to revisit the ReadMe as well, which is the landing and more of a "selling" point.

    

#### 9.14.1. Regenerate the md docs


    # instead of the product instance dir you could also use different documents root 
    export docs_root_dir=/hos/opt/csitea/qto/qto.0.7.7.dev.ysg
    # Action !!!
    bash src/bash/qto/qto.sh -a generate-md-docs

#### 9.14.2. Regenerate the pdf docs
Regenerate the pdf docs by issuing the following one-liner

    # instead of the product instance dir you could also use different documents root 
    export docs_root_dir=/hos/opt/csitea/qto/qto.0.7.7.dev.ysg
    
    bash src/bash/qto/qto.sh -a generate-pdf-docs

#### 9.14.3. Regenerate the msft docs
Regenerate the Microsoft docx files by issuing the following shell one-liner:

    # instead of the product instance dir you could also use different documents root 
    export docs_root_dir=/hos/opt/csitea/qto/qto.0.7.7.dev.ysg
    # Action !!!
    bash src/bash/qto/qto.sh -a generate-msft-docs

### 9.15. The related requirement is added in the requirements document
The related requirement is added in the requirements document - there might be one or more requirements added. 

    

### 9.16. At least 2 times passed functional and js tests run 
Use the following shell actions ( Note that since v0.6.7 as authentication is required for most of the web-actions the QTO_ONGOING_TEST environmental variable has to be set to 1 to run those separately) :

    bash src/bash/qto/qto.sh -a run-js-tests
    bash src/bash/qto/qto.sh -a run-functional-tests

### 9.17. At least 2 times passed integration tests  in each environment instance
At least 2 times passed unit tests run in each environment instance - run the unit tests at least twice per environment. Should the run behave differently start all over from dev. Since v0.6.7 as authentication is required for most of the web-actions the QTO_ONGOING_TEST environmental variable has to be set to 1 to run those separately. 

    bash src/bash/qto/qto.sh -a run-integration-tests

### 9.18. Deployment to the test environment
Deploy to the test environment as shown in the code snippet below. Re-run the tests via the tests shell actions. 

    # deploy to the tst environment
    bash src/bash/qto/qto.sh -a to-tst
    
    # go to the product instance dir of the tst env for this version
    cd ../qto.<<version>>.tst.<<owner>>
    

### 9.19. Check that all the files in the deployment package are the same as those in the latest commit of the dev git branch. 
Deploy to the test environment as follows:

    # deploy to the tst environment
    bash src/bash/qto/qto.sh -a to-tst
    
    # go to the product instance dir of the tst env for this version
    cd ../qto.<<version>>.tst.<<owner>>
    

### 9.20. restart the application layer
Well just chain the both commands. 

    bash src/bash/qto/qto.sh -a mojo-morbo-stop ; bash src/bash/qto/qto.sh -a mojo-morbo-start

## 10. DEVBOX SETUP ( optional )
Every developer has his/her own development setup - this section is completely optional - your setup might be much more developed than the one suggested here ... You might find it 

    # run the automatic personal setup
    curl https://raw.githubusercontent.com/YordanGeorgiev/ysg-confs/master/src/bash/deployer/setup-bash-n-vim.sh | bash -s <EMAIL>

### 10.1. The tmux terminal multiplexer
In this setup the tmux.conf contains all the instructions on the shortcut mappings, which enable quick navigation from one UI terminal window to multiple sessions with multiple windows etc. ...

    cat ~/.tmux.conf

### 10.2. The vim IDE
Once again we DO NOT encourage "religious wars", if you happen to be using or interested about the setup used for development, check the .vimrc at the root of the project, which will point you to all the plugins, shortcuts and settings used.

    # to check the syntax in the current file:
    :make
    
    # to check the syntax in the whole project from the proj root:
    bash ./src/bash/qto/qto.sh -a check-perl-syntax

## 11. DEBUGGING
Debugging is bad - it almost means that you have already lost the battle with the complexity of the code and MUST slow down to grasp what is actually happening ... 
But .. if you are already here than use the following hints ...

    

### 11.1. Enable debugging in Mojolicious
Set the env var(s) and restart morbo.
Check the following list of the Mojolicous env vars:
https://github.com/mojolicious/mojo/wiki/%25ENV

    # stop the application layer run-time if it is already running
    bash src/bash/qto/qto.sh -a mojo-morbo-stop
    
    # set the env var
    export MOJO_MODE=development
    # ... or any other of the variables above ...
    
    # restart the application layer 
    bash src/bash/qto/qto.sh -a mojo-morbo-start

### 11.2. Enable debugging in DBI
Set the DBI_TRACE env variable and restart the application layer. Check the dbitrace.log for your exact statement.

    # stop the application layer run-time if it is already running
    bash src/bash/qto/qto.sh -a mojo-morbo-stop
    
    # set the env var
    DBI_TRACE=1=dbitrace.log
    export DBI_TRACE
    
    # restart the application layer 
    bash src/bash/qto/qto.sh -a mojo-morbo-start

### 11.3. Debugging in the UI
The  mojo *html.ep DO contain a lot of JavaScript code, which IS RUN on the client AFTER it is run on the server side, thus the good old console.log or console.error work ... Quite often it makes sense to debug large JS objects, especially those related to VueJS.


    console.log(this.$parent)
    console.log("what exactly my parent control contains")
    console.log("todo:<<me>> remember to remove this ^^^")
    // ^^^ this one might seem like overkill , BUT you will be amazed
    // how-often you will have more than 5 console.log statements, which 
    // you would have to hunt down, after the completion of the feature by:
    find . -type f -name '*.html.ep' -exec grep -nHi console.log {} \;

### 11.4. Debugging css
Css editing IS time consuming activity. IMPORTANT !!! Chrome checkbox in the Networks "Disable cache" is buggy and unreliable - the ONLY way to clear the cache of a site you are performing css changes to actually see the changes is : 
 - F12 - to open DevTools
 - right-click on the reload , button and choose - "Clear cache and Hard Reload "
The other alternative is to open a new tab with the same url from inkognito mode ( which is also proven to be not 100% reliable ... )

    

## 12. SECURITY


    

### 12.1. Authentication
You might want to refresh the following security related links from time while reading this section:

http://self-issued.info/docs/draft-jones-json-web-token-06.html
https://metacpan.org/pod/Mojo::JWT
https://tools.ietf.org/html/rfc6749#section-1.5

    

#### 12.1.1. JWT based native authentication
Theory chk the following links:
http://self-issued.info/docs/draft-jones-json-web-token-06.html
https://metacpan.org/pod/Mojo::JWT
https://tools.ietf.org/html/rfc6749#section-1.5

    

#### 12.1.2. Session based native authentication
The session based authentication works basically as follows:
 - non-authenticated users requests a resource from the application layer
 - the application layer , runs the controller specified in the route
 - each controller is derived from the BasedController, which has the isAuthenticated metho
 - which returns 1 or 0 based on Mojo::Session stored data

So as of v0.7.8 - no app_roles, no permissions are implemented - the users are either authenticated or not. Once authenticated they can CRUD anything they have access to from the UI.

    

#### 12.1.3. RBAC based native authentication
The RBAC based native authentication works as follows:
- during start-up or meta-data reload the Guardian component saves the RBAC list into the Redis
- the User authenticates against the System via the login
- The System grants the list of app_roles to the JWT token of the user

    

### 12.2. Authorisation
As of v0.8.1 the Roles-Based Access Control List is being implemented. You might want to refresh your RBAC theoretical skills:
https://searchsecurity.techtarget.com/definition/role-based-access-control-RBAC

    

#### 12.2.1. Generic role-based access control list based authorisation
Start by defining your app_roles in the list/app_roles page. Keep the number of app_roles in the beginning to the absolute minimum - you could easily re-do the whole RBAC list re-population later on.
Who has access to what is defined in the following table: app_items_roles_permissions. 
You could initially load this table by running the following scripts below. 
https://stackoverflow.com/a/58009983/65706

    # populate the list of tables into the app_items table
    psql -d dev_qto < /src/sql/pgsql/scripts/admin/populate-app_items.sql
    
    # populate the items app_roles_permissions
    psql -d dev_qto < src/sql/pgsql/scripts/admin/populate-app_items_roles_permissions.sql

## 13. KNOWN ISSUES AND WORKAROUNDS


    

### 13.1. The RBAC based native authorisation
The RBAC based native authentication works as follows:
 - during start-up or meta-data reload the Guardian component saves the RBAC list into the Redis
 - the User requests a resource from the System
 - The Guardian component takes the role claims from the Users JWT
 - The Guardian component builds the role-page resource id and checks that it exists from the RBAC list in Redis
 - if the request role-page resources id exists the Guardian passes the User to fetch the resource
 - if the request role-page resource id DOES NOT exist : 
    -- the Guardian redirects the user to the login page if the user is not authenticated
    -- the Guardian redirect the user to the search / home pager if the user is authenticated

    

### 13.2. All tests fail with the 302 error
 This one is actually a bug ... all the tests not requiring non-authentication mode should set it in advance ...

    # disable authentication during testing
    export QTO_NO_AUTH=1
    
    # call the test once again
    perl src/perl/qto/t/lib/Qto/Controller/TestHiCreate.t

### 13.3. Morbo is stuck


    

#### 13.3.1. Probable root cause
This one occurs quite often , especially when the application layer is restarted, but the server not 

    # the error msg is 
     [INFO ] 2018.09.14-10:23:14 EEST [qto][@host-name] [4426] running action :: mojo-morbo-start:doMojoMorboStart
    (Not all processes could be identified, non-owned process info
     will not be shown, you would have to be root to see it all.)
    tcp        0      0 0.0.0.0:3001            0.0.0.0:*               LISTEN      6034/qto
    tcp        0      0 0.0.0.0:3002            0.0.0.0:*               LISTEN      7626/qto
    Can't create listen socket: Address already in use at /usr/local/share/perl/5.26.0/Mojo/IOLoop.pm line 130.
     [INFO ] 2018.09.14-10:23:16 EEST [qto][@host-name] [4426] STOP FOR qto RUN with:
     [INFO ] 2018.09.14-10:23:16 EEST [qto][@host-name] [4426] STOP FOR qto RUN: 0 0 # = STOP MAIN = qto
    qto-dev ysg@host-name [Fri Sep 14 10:23:16] [/vagrant/opt/csitea/qto/qto.0.4.9.dev.ysg] $

#### 13.3.2. Kill processes
List the running perl processes which run the morbo and kill the instances

    ps -ef | grep -i perl
    
    # be carefull what to kill 
    kill -9 <<proc-I-know-is-the-one-to-kill>>

#### 13.3.3. Problem description
This one occurs quite often , especially when the application layer is restarted, but the server not 

    # the error msg is 
     [INFO ] 2018.09.14-10:23:14 EEST [qto][@host-name] [4426] running action :: mojo-morbo-start:doMojoMorboStart
    (Not all processes could be identified, non-owned process info
     will not be shown, you would have to be root to see it all.)
    tcp        0      0 0.0.0.0:3001            0.0.0.0:*               LISTEN      6034/qto
    tcp        0      0 0.0.0.0:3002            0.0.0.0:*               LISTEN      7626/qto
    Can't create listen socket: Address already in use at /usr/local/share/perl/5.26.0/Mojo/IOLoop.pm line 130.
     [INFO ] 2018.09.14-10:23:16 EEST [qto][@host-name] [4426] STOP FOR qto RUN with:
     [INFO ] 2018.09.14-10:23:16 EEST [qto][@host-name] [4426] STOP FOR qto RUN: 0 0 # = STOP MAIN = qto
    qto-dev ysg@host-name [Fri Sep 14 10:23:16] [/vagrant/opt/csitea/qto/qto.0.4.9.dev.ysg] $

### 13.4. The page looks broken - probably the new css is not reloaded
This problem is quite oftenly experienced and a real time-burner, so keep those shortcuts below in mind. 
To apply the newest css do a hard reload in Chrome with the shortcut COMMAND + SHIFT + R.
The other option is to keep the SHIFT button and press the reload button the Chrome address bar ( this one has been buggy from time to time as well. ... )

    COMMAND + SHIFT + R
    SHIT + CLICK ON RELOAD BUTTON

### 13.5. The vue UI does not update properly 
Due to the vue reactivity system - basically the more the complex the ui control, the better to have some kind of id to set in the v-for ... 

    <div v-for="item in filteredItems" :key="item.id">
    
    // use also in every CRUD operation ...
    this.$forceUpdate()
    // or even 
    this.$parent.$forceUpdate()

### 13.6. NginX fails with 502 Bad Gateway error and probably crashes the site
Might be due to the following error found in the journal log: 
"nginx.service: Failed to parse PID from file /run/nginx.pid: Invalid argument"

    sudo mkdir /etc/systemd/system/nginx.service.d
    printf "[Service]\
    ExecStartPost=/bin/sleep 0.1\
    " | \
        sudo tee /etc/systemd/system/nginx.service.d/override.conf
    sudo systemctl daemon-reload
    sudo systemctl restart nginx

### 13.7. Why having all the hassle with this directory structure - is overkill ?!!
Every software project has a scope. Qto as a project aims to provide the fully vertically integrated code base for deployment, operation and maintenance, which is not possible without the use of multiple run-times within the project ( such as bash, perl, python, terraform, npm ... )
Having a project with a directory structure for a specific run-time enforcing that directory structure to all the other runtimes is a a mess.

    

### 13.8. The new entries in DropBox FK tables cannot be seen
I updated a FK table - the new entries are there. I can see from db that the correct entries are in the db , but I cannot still see the new value(s) in the drop down ... 
It is a Google Chrome specific cache bug ... Try first to hard reload and open the browser , Inspect ( DevTools has to be open ) right-click the Reload button on the address bar and choose "Hard Reload and empty cache" or CMD + Option + R shortcut combination. 
If that STILL does not work on the DevTools choose the Application menu, Storage , Local storage , right-click the local storage of the instance the problem occurs and choose clear local storage.

    

## 14. FAQ
This section contains the most probable frequently asked questions.

    

<file_sep>-- DROP TABLE IF EXISTS ideas ; 

SELECT 'create the "ideas" table'
; 
   CREATE TABLE ideas (
    guid uuid DEFAULT public.gen_random_uuid() NOT NULL,
    id bigint DEFAULT (to_char(CURRENT_TIMESTAMP, 'YYMMDDHH12MISS'::text))::bigint NOT NULL,
    app_users_guid uuid DEFAULT '2660a6e9-9e6b-4faa-8264-27a92872657b'::uuid NOT NULL,
    prio integer DEFAULT 0,
    ideas_status_guid uuid DEFAULT 'cb989a14-d0b8-46e4-b2cc-5e2a974b5d29'::uuid NOT NULL,
    name character varying(100) DEFAULT 'name ...'::character varying NOT NULL,
    description character varying(4000) DEFAULT 'description ...'::character varying NOT NULL,
    owner character varying(50) DEFAULT 'unknown'::character varying NOT NULL,
    update_time timestamp without time zone DEFAULT date_trunc('second'::text, now())
    , CONSTRAINT pk_ideas_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

create unique index idx_ideas_uniq_id on ideas (id);


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.ideas'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 

--The trigger:
CREATE TRIGGER trg_set_update_time_on_ideas BEFORE UPDATE ON ideas FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'ideas'::regclass;
<file_sep>UNAME := $(shell uname)
ifeq ($(UNAME),Darwin)
	OS_X   := true
	SHELL  := /bin/bash
else
	OS_DEB := true
	SHELL  := /bin/bash
endif
<file_sep>#!/usr/bin/env bash

export TERM=xterm

print_ok() {
    GREEN_COLOR="\033[0;32m"
    DEFAULT="\033[0m"
    echo -e "${GREEN_COLOR} ✔ [OK] ${1:-} ${DEFAULT}"
}

print_fail() {
    RED_COLOR="\033[0;31m"
    DEFAULT="\033[0m"
    echo -e "${RED_COLOR} ❌ [NOK] ${1:-}${DEFAULT}"
}
<file_sep>#!/usr/bin/env python3
# purpose: a quick and dirty semi-generic template generator, based on dir naming convention and
# shallow data structure ...
# usage: src/python/tpl_gen.py

import sys
import os
import os.path as path
import json
import glob
from jinja2 import Environment, BaseLoader
from pprintjson import pprintjson as ppjson
import pprint

product_instance_dir, json_cnf_fle, cnf, pp, version = '', '', '', '', ''

def main():
    set_vars()
    do_generate(cnf)
    sys.exit(0)

def set_vars():
    try:
        global product_instance_dir, json_cnf_fle, cnf, pp, cnf, version
        pp = pprint.PrettyPrinter(indent=3)
        product_instance_dir = path.abspath(path.join(__file__ ,"../../.."))
        do_read_conf_fle(product_instance_dir + '/.env') # env agnostic
        env = os.environ['ENV']
        version = os.environ['VERSION']

        with open(product_instance_dir + '/cnf/env/' + env + '.env.json') as json_cnf_fle:
            cnf = json.load(json_cnf_fle)
            cnf['env']['VERSION'] = version
            cnf['env']['VER'] = version.replace('.','')
        ppjson ( cnf )

    except(IndexError) as error:
        print ("ERROR in set_vars: " , str(error)) 
        traceback.print_stack()
        sys.exit(1)


def do_read_conf_fle(f):
    try:
        with open(f, 'r') as fh:
            vars_dict = dict(
                tuple(line.rstrip().split('='))
                for line in fh.readlines() if not line.startswith('#')
            )
        os.environ.update(vars_dict)
    except (Exception) as error:
        print('ERROR in do_read_conf_fle:' , error)
        traceback.print_stack()
    finally:
        print("RUNNING in the following env: " , vars_dict)


def do_generate(cnf):
    for f in glob.iglob( product_instance_dir + '/src/tpl/**/*.tpl', recursive=True):
        try:
            print ( "read template file: " , f)
            pp.pprint (cnf)
            str_tpl = open(f, 'r').read()
            obj_tpl = Environment(loader=BaseLoader).from_string(str_tpl)
            rendered = obj_tpl.render(cnf['env'])
            print(cnf)
            pp.pprint ( rendered )
            tgt_fle = f.replace('/src/tpl','',1).replace('.tpl','')
            # print (rendered)
            print(rendered,  file=open(tgt_fle, 'w'))
            print ( "output ready rendered file : " , tgt_fle)
        except Exception as e:
            print ("RENDERING EXCEPTION: \n", str(e))
            traceback.print_stack()
    print("STOP generating templates")


main()
<file_sep># file: cnf/crontab/crontab-l.sh
# usage:
# to-deploy this configuration
# crontab cnf/crontab/crontab-l.sh

# every day at 03:00 perform a full db backup in TST
1 3 * * * bash /home/ubuntu/opt/qto/qto.0.8.3.tst.ubuntu@qto.fi/src/bash/qto/qto.sh -a backup-postgres-db -a backup-postgres-db-inserts -a publish-dbdump-to-s3

# every day at 03:15 perform a full restart of the application layer in TST

# every day at 03:15 perform a full restart of the application layer in PRD
# fails !!! qto-200325185208
# 15 3 * * * bash /home/ubuntu/opt/qto/qto.0.8.3.prd.ubuntu@qto.fi/src/bash/qto/qto.sh -a mojo-hypnotoad-start
15 3 * * * sudo service nginx restart

# every day at 19:00 perform a full db backup in TST
1 19 * * * bash /home/ubuntu/opt/qto/qto.0.8.3.tst.ubuntu@qto.fi/src/bash/qto/qto.sh -a backup-postgres-db -a backup-postgres-db-inserts -a publish-dbdump-to-s3

#
# 0 3 * * *
# * * * * * <<command-to-be-executed>>
# - - - - -
# | | | | |
# | | | | +- - - - day of week (0 - 6) (Sunday=0)
# | | | +- - - - - month (1 - 12)
# | | +- - - - - - day of month (1 - 31)
# | +- - - - - - - hour (0 - 23)
# +--------------- minute

# 
# edit the crontab
# export EDITOR=vim; sudo crontab -e
# 
# to edit the crontab in batch mode
# crontab -l | sed 's/>/>>/' | crontab -
# 
# to view the crontab
# sudo crontab -l
# 
# when in doubt chk also: https://crontab.guru/#0_19_*_*_*
# eof file: cnf/crontab/crontab-l.sh
<file_sep>do_create_app_db(){

	# set up confs
   source $PRODUCT_DIR/.env ; ENV=$ENV
   test -z ${PROJ_CONF_FILE:-} && export PROJ_CONF_FILE="$PRODUCT_DIR/cnf/env/$ENV.env.json"
   source $PRODUCT_DIR/lib/bash/funcs/export-json-section-vars.sh
   do_export_json_section_vars $PROJ_CONF_FILE '.env.db'
   
   # create the app db
   tmp_log_file="/tmp/.$$.log"
	pgsql_scripts_dir="$PRODUCT_DIR/src/sql/pgsql/qto"
   sql_script="$pgsql_scripts_dir/00.create-db.pgsql"

   PGPASSWORD="${<PASSWORD>_sys_usr_<PASSWORD>:-}" psql -v -q -t -X -w -U "${postgres_sys_usr_admin:-}" \
      -h $postgres_rdbms_host -p $postgres_rdbms_port -v ON_ERROR_STOP=1 \
		-v postgres_app_db="${postgres_app_db:-}" \
      -f "$sql_script" postgres > "$tmp_log_file" 2>&1
   ret=$?

   cat "$tmp_log_file" ; cat "$tmp_log_file" >> $log_file # show it and save it 
   test $ret -ne 0 && { 
      sleep 3 ; do_exit 1 "pid: $$ psql ret $ret - failed to run sql_script: $sql_script !!!" ; break ;
   }
  
	# verify the sql for the provisioning of the postgres_sys_usr_admin
  sudo -Hiu postgres PGPASSWORD=$postgres_<PASSWORD> psql --port $postgres_rdbms_port \
         --host $postgres_rdbms_host -d postgres -c '\l'
  sleep 1
}
<file_sep>#!/usr/bin/env python
# vim: ts=4 sw=4 et

# https://github.com/ilyash/show-struct.git
from __future__ import unicode_literals
import argparse
import collections
import json
import sys


class Outliner(object):

    def __init__(self):
        self.paths = {}
        self.values_for_path = collections.defaultdict(dict)

    def _outline(self, data, path):
        p = ''.join(path)
        self.paths[p] = True
        if isinstance(data, dict):
            if not data:
                self.values_for_path[p]['(Empty hash)'] = True
            for k, v in data.iteritems():
                self._outline(v, path + ['.' + k])
            return
        if isinstance(data, list):
            for v in data:
                self._outline(v, path + ['[]'])

            sentence = '(Array of {0} elements)'.format(len(data))
            self.values_for_path[p][sentence] = True
            return
        # scalar assumed
        self.values_for_path[p][data] = True

    def outline(self, data):
        self._outline(data, [])
        del self.paths['']

        ret = []
        for path in sorted(self.paths):
            ret.append({
                'path': path,
                'values': sorted(self.values_for_path[path].keys())
            })
        return ret


if __name__ == '__main__':
    parser = argparse.ArgumentParser('Show structure of give JSON file')
    parser.add_argument('file', metavar='FILE', default='-', nargs='?',
                        help="The filename to read. Use '-' to read stdin - defaults to '-'")
    args = parser.parse_args()

    if args.file == "-":
        data = json.loads(sys.stdin.read())
    else:
        with open(args.file) as f:
            data = json.loads(f.read())

    outline = Outliner().outline(data)
    for path in outline:
        l = len(path['values'])
        if l == 0:
            print(path['path'])
            continue
        if l == 1:
            print("{0} -- {1}".format(path['path'], path['values'][0]))
            continue
        print("{0} -- {1} .. {2} ({3} unique values)".format(
            path['path'],
            path['values'][0],
            path['values'][-1],
            l,
        ))
<file_sep>#!/bin/bash
#------------------------------------------------------------------------------
# usage example:
# source lib/bash/funcs/require-var.func.sh
# do_require_var ENV $ENV ; do_require_var ORG $ORG ;
#------------------------------------------------------------------------------
do_require_var(){

  var_name="${1:-}"
  var="${2:-}"

  do_simple_log(){
    type_of_msg=$(echo $*|cut -d" " -f1)
    msg="$(echo $*|cut -d" " -f2-)"
    echo " [$type_of_msg] `date "+%Y-%m-%d %H:%M:%S %Z"` [$$] $msg "
  }

  test -z "${var:-}" && {
    do_simple_log 'FATAL The environment variable "'$var_name'" does not have a value !!!'
    do_simple_log 'INFO In the calling shell do "export '$var_name'=your-'$var_name'-value"'
    exit 1
  }
}
<file_sep>  ---- DROP TABLE IF EXISTS skills ; 

SELECT 'create the "skills" table'
; 
   CREATE TABLE skills (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , seq            integer NULL
    , prio           integer NULL
    , name           varchar (100) NOT NULL DEFAULT 'name ...'
    , description    varchar (4000) NOT NULL DEFAULT 'description ...'
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_skills_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

create unique index idx_skills_uniq_id on skills (id);


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.skills'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 

--The trigger:
CREATE TRIGGER trg_set_update_time_on_skills BEFORE UPDATE ON skills FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'skills'::regclass;
<file_sep>doUncrambleConfs(){
   while read -r file ; do
      test -f $file.bak || continue
      test -f $file.bak && cp -v $file.bak $file
      test -f $file.bak && rm -v $file.bak
	done < <(find $PRODUCT_DIR/cnf/env/ -type f -name '*.json')
   exit 0
}
<file_sep># src/bash/qto/funcs/clone-project.help.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doHelpCloneProject comments ...
# ---------------------------------------------------------
doHelpCloneProject(){

	do_log "DEBUG START doHelpCloneProject"
	
	cat doc/txt/qto/helps/clone-project.help.txt
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!

	do_log "DEBUG STOP  doHelpCloneProject"
}
# eof func doHelpCloneProject


# eof file: src/bash/qto/funcs/clone-project.help.sh
<file_sep># src/bash/qto/funcs/xls-to-db.func.sh

# v0.1.8
# ---------------------------------------------------------
# load xls to db
# cat doc/txt/qto/funcs/xls-to-db.func.txt
# ---------------------------------------------------------
doXlsToDb(){

	do_log "DEBUG START doXlsToDb"
	
	sleep "$sleep_interval"
   test -z ${items_order_by_attribute+x} && export items_order_by_attribute='category'
   test -z ${period+x} && export period='daily'
   test -z ${tables+x} && export tables='daily_issues'
   
   # find out the latest xls file from the project daily dir
   # pass it to the xls-to-rdbms tool as the input xls file
   # or leave the app to guess it ... 
   test -z ${xls_file+x} && \
      export xls_file=$(find ${mix_data_dir} -name '*.xlsx'| grep '.all.'| sort -rn|head -n 1)
   test -z ${xls_file} && \
      export xls_file=$(find $mix_data_dir -name '*.xlsx'| grep $period| sort -rn|head -n 1)

   cmd="perl src/perl/qto/script/qto.pl --do xls-to-db --tables $tables"
   test -z ${xls_file} || cmd="$cmd --xls-file $xls_file"

   do_log "INFO using: mix_data_dir: ${mix_data_dir+}"
   do_log "INFO using xls_file: $xls_file"
   # Action !!!
   $cmd

   exit_code=$?
   
#   psql -d "$postgres_app_db" -c '
#   SELECT id category , substring ( description from 0 for 40 ) as descrption , start_time , stop_time
#   FROM '"$tables"'_issues order by '"$items_order_by_attribute"'
#   ;';

   do_log "INFO doRunQto exit_code $exit_code"
   test $exit_code -ne 0 && do_exit $exit_code "failed to run qto.pl"  
  
	do_log "DEBUG STOP  doXlsToDb"
}
# eof func doXlsToDb


# eof file: src/bash/qto/funcs/xls-to-db.func.sh
<file_sep># v0.5.8
# ---------------------------------------------------------
#	run all the js 
# ---------------------------------------------------------
doRunJsUnitTests(){
	which mocha 2>/dev/null || \
		{ 
         sudo npm install -g --save-dev mocha
         echo >&2 "The mocha is missing - \"sudo npm install -g --save-dev mocha \" ! Aborting ..."; 
         exit 1; 
      }
   # sudo npm install -g --save-dev chrome-remote-interface
	# which chai 2>/dev/null || \
	 #  { echo >&2 "The chai is missing - \"sudo npm install -g --save-dev chai \" ! Aborting ..."; exit 1; }
	source $PRODUCT_DIR/lib/bash/funcs/flush-screen.sh

	js_unit_tests_dir="$PRODUCT_DIR/src/js/node/js-unit-tests"
	cd $js_unit_tests_dir
	while read -r d ; do
		echo -e "cd to dir: $d"
		cd $d
		echo -e "installing packages for dir $d: \n"
		sleep 1
		npm install
		do_flush_screen  # clear the screen
		echo -e "test dir $d \n"
		sleep 1
		npm test
		sleep 4
		do_flush_screen
	done < <(find $js_unit_tests_dir -type d -name '0*-*'|sort)
}
<file_sep># source & courtesy of:
# https://github.com/mislav/dotfiles/blob/master/bin/tmux-session


# v0.5.8
# ---------------------------------------------------------
# restores a tmux session
# ---------------------------------------------------------
doRestoreTmuxSession(){
   source "$PRODUCT_DIR/src/bash/qto/funcs/sys/tmux/tmux-common.sh"
   doCheckTmuxIsInstalled
   set +e
   local count=0
   local dimensions="$(terminal_size)"

   while IFS=$'\t' read session_name window_name dir; do
    if [[ -d "$dir" && $window_name != "log" && $window_name != "man" ]]; then
      # echo $session_name -- $window_name
      if session_exists "$session_name"; then
        add_window "$session_name" "$window_name" "$dir"
      else
        new_tmux_session "$session_name" "$window_name" "$dir" "$dimensions"
        count=$(( count + 1 ))
      fi
    fi
   done < ~/.tmux-session

   tmux list-sessions | column -t | sort

}
#eof func doRestoreTmuxSession
<file_sep># src/bash/qto/funcs/start-selenium.func.sh

# v1.0.9
# ---------------------------------------------------------
# cat doc/txt/qto/funcs/start-selenium.func.txt
# ---------------------------------------------------------
doStartSelenium(){

	do_log "DEBUG START doStartSelenium"
	
   selenium='/usr/lib/selenium-server-standalone.jar'
   chrome_driver=$(which chromedriver)

   test -f $(which java) || export exit_code=1 ; do_exit 1 " java is not installed. v1.8 required "
   test -f "$selenium" || export exit_code=1 ; do_exit 1 " the selenium jar $selenium is not installed" 
   test -f "$chrome_driver" || export exit_code=1 ; do_exit 1 " the chrome driver: $chrome_driver is not installed"
	
	sleep "$sleep_interval"
   echo "clean-up before start "
   rm -rv /tmp/.X99-lock
   rm -vr /tmp/.*-lock
   rm -rv /tmp/.X*

   echo check for already running instances 
   ps -ef | grep -i java 

	# Action !!!
   xvfb-run -e /dev/stdout java -Dwebdriver.chrome.driver=$chrome_driver -jar "$selenium" &

	do_log "DEBUG STOP  doStartSelenium"
}
# eof func doStartSelenium

# eof file: src/bash/qto/funcs/start-selenium.func.sh
<file_sep># src/bash/qto/funcs/remove-action-files.func.sh

# v1.1.2
# ---------------------------------------------------------
# obs we assume that the caller is in the PRODUCT_DIR
# simply delete each file which greps finds to match to the action
# name(s) configured in the : 
# src/bash/qto/tests/rem-qto-actions.lst
# list file
# ---------------------------------------------------------
doRemoveActionFiles(){

	do_log "DEBUG START doRemoveActionFiles"
	
	# for each defined action 	
	while read -r act ; do (
		do_log "INFO STOP  :: removing action: $act"
		find . | grep $act | cut -c 3- | xargs rm -fv "{}"
      for env in `echo dev tst prd src`; do perl -pi -e 's/^.*?'$act'.*\n$//gm' "met/.$env.";done;
		
	); 
	done< <(cat "src/bash//tests/rem--actions.lst")

	do_log "DEBUG STOP  doRemoveActionFiles"
}
# eof func doRemoveActionFiles


# eof file: src/bash/qto/funcs/remove-action-files.func.sh
<file_sep>#!/bin/bash
do_print_usage(){
   # if $run_unit is --help, then message will be "--help deployer PURPOSE"
   cat << EOF_USAGE
   :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
   This is a generic bash funcs runner script:
   :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

   You can also execute one or multiple actions with the
   $0 --action <<action-name>>
   or
   $0 -a <<action-name>> -a <<action-name-02>>

   where the <<action-name>> is one of the following

EOF_USAGE

   find src/bash/run/ -name *.func.sh \
      | perl -ne 's/(.*)(\/)(.*).func.sh/$3/g;print'| perl -ne 's/-/_/g;print "do_" . $_' | sort

   exit 1
}
<file_sep>-- v0.8.4

-- \echo 'If necessary, perform -- DROP TABLE IF EXISTS app_items;'

-- \echo '5. Creating the app_items table'

   CREATE TABLE app_items (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigserial UNIQUE 
    , prio           smallint NOT NULL DEFAULT 1
    , name           varchar (200) NOT NULL DEFAULT 'Item name'
    , ver            varchar (10) NOT NULL DEFAULT '0.8.4'
    , description    varchar (4000) DEFAULT ''
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_app_items_guid PRIMARY KEY (guid)
    );

CREATE UNIQUE INDEX idx_uniq_app_items_id
	ON app_items (id);

-- \echo 'List columns of the created table app_items'
SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid='public.app_items'::regclass
   AND    attnum>0
   AND    NOT attisdropped
   ORDER  BY attnum; 

-- \echo 'Update time on every EXECUTE trigger:'
CREATE TRIGGER trg_set_update_time_on_app_items
	BEFORE UPDATE ON app_items
	FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
	FROM pg_trigger
	WHERE NOT tgisinternal
	AND tgrelid='app_items'::regclass;

--
-- TOC entry 3310 (class 0 OID 58960)
-- Dependencies: 207
-- Data for Name: app_items; Type: TABLE DATA; Schema: public; Owner: usrtstqtoadmin
--

COPY public.app_items (guid, id, prio, name, ver, description, update_time) FROM stdin;
1948aa5f-3007-4b0a-8c59-b1a5b5fbcff1	1	1	benchmarks	0.8.3	benchmarks	2020-04-12 18:14:53
71fc6f55-32f5-4ec9-ae7b-d552d6196ec8	2	1	category	0.8.3	category	2020-04-12 18:14:53
149a194d-b20e-44d5-b1a5-4fbd32e058ce	3	1	check_lists	0.8.3	check_lists	2020-04-12 18:14:53
bd992a20-1f72-48d6-a89c-5f321d812b76	4	1	concepts_doc	0.8.3	concepts_doc	2020-04-12 18:14:53
fc8c1ece-5804-4ea7-9d50-c4778f18cbe8	5	1	confs	0.8.3	confs	2020-04-12 18:14:53
b59fdeca-3eac-4cbb-93f7-13e488ec993e	6	1	daily_issues	0.8.3	daily_issues	2020-04-12 18:14:53
6bcf5612-c611-42f7-91d7-499e4abe3e04	7	1	decadally_issues	0.8.3	decadally_issues	2020-04-12 18:14:53
5347e5b6-c366-4779-9592-b85caadb4be7	9	1	devops_guide_doc	0.8.3	devops_guide_doc	2020-04-12 18:14:53
eca33c21-138c-4a2f-b907-3b1d117bea64	8	1	definitions_dictionary	0.8.3	definitions_dictionary	2020-04-12 18:14:53
a0149328-0a2f-42b1-8725-714233c40062	10	1	enduser_guide_doc	0.8.3	enduser_guide_doc	2020-04-12 18:14:53
12660a04-8e58-4094-acf2-477a58f524e8	11	1	export_files	0.8.3	export_files	2020-04-12 18:14:53
532a3298-e92d-4171-83fe-87402220f03a	12	1	features_doc	0.8.3	features_doc	2020-04-12 18:14:53
7a99c707-da32-43ac-88fb-475255ff6a56	13	1	goals	0.8.3	goals	2020-04-12 18:14:53
c26dd0e6-c934-4511-80ee-4e58406f47d0	14	1	his_quinquennially_issues	0.8.3	his_quinquennially_issues	2020-04-12 18:14:53
c8cb693b-1b96-49ff-8f24-6e4ea9dca2cb	15	1	ideas	0.8.3	ideas	2020-04-12 18:14:53
8683aa99-d848-4f36-99a9-0a6632a946d8	16	1	ideas_status	0.8.3	ideas_status	2020-04-12 18:14:53
aba02ad6-4d51-4ce7-8635-cad777b0e520	17	1	imgs	0.8.3	imgs	2020-04-12 18:14:53
33a1dc14-323f-4612-a899-6bc2c3518739	18	1	installations_doc	0.8.3	installations_doc	2020-04-12 18:14:53
8d2249c4-ec14-43e8-ac26-49699cd0cdff	19	1	issues_status	0.8.3	issues_status	2020-04-12 18:14:53
6c1c3bae-e51f-4fba-9f01-d41991d08be8	20	1	items_doc	0.8.3	items_doc	2020-04-12 18:14:53
4cd69b6b-a6ed-49f8-8a19-90fdf16201d7	21	1	app_items_roles_permissions	0.8.3	app_items_roles_permissions	2020-04-12 18:14:53
a2b659fc-5b8b-432e-a912-e3948eeee48a	22	1	links	0.8.3	links	2020-04-12 18:14:53
4e2523dc-8c74-4130-9c77-ed7a81ca456a	23	1	logs	0.8.3	logs	2020-04-12 18:14:53
60d46031-2164-42d6-b51c-16c0e801f5f1	24	1	maintenance_guide_doc	0.8.3	maintenance_guide_doc	2020-04-12 18:14:53
704c4d59-8cac-492f-a2d1-510e6dea888f	25	1	meta_cols	0.8.3	meta_cols	2020-04-12 18:14:53
3f839b5d-ad22-47d1-92df-e1cca19545fa	26	1	app_item_attributes	0.8.3	app_item_attributes	2020-04-12 18:14:53
1c1f8263-5451-42e1-aab6-7ba77a96d772	27	1	app_routes	0.8.3	app_routes	2020-04-12 18:14:53
c1773113-a314-4813-a1a3-15a10ad8bcc5	28	1	app_items	0.8.3	app_items	2020-04-12 18:14:53
384fbb48-3c4d-465f-b018-47797d2c674d	32	1	naming_conventions	0.8.3	naming_conventions	2020-04-12 18:14:53
44c0e077-1fc5-4719-afef-5d346458772e	33	1	nix_book_doc	0.8.3	nix_book_doc	2020-04-12 18:14:53
5b7a0d36-2c47-4bcf-91cf-9bb8a20627d6	34	1	onboarding_guide_doc	0.8.3	onboarding_guide_doc	2020-04-12 18:14:53
0b264bef-93c3-494a-a984-ef94ed673608	35	1	principles	0.8.3	principles	2020-04-12 18:14:53
7a51bf56-9a79-41af-b2be-bce883a5638c	36	1	problems	0.8.3	problems	2020-04-12 18:14:53
56278003-869f-431e-a607-7642bca24a61	37	1	questions	0.8.3	questions	2020-04-12 18:14:53
102898fb-2819-4608-85f4-1b1ff7417de3	38	1	questions_status	0.8.3	questions_status	2020-04-12 18:14:53
7612f252-e3f4-4327-be6a-3e57545a1cc4	39	1	quinquennially_issues	0.8.3	quinquennially_issues	2020-04-12 18:14:53
2ef44587-2708-45e5-9a7d-12dc49fc2c0a	40	1	quinquennially_issues_2020	0.8.3	quinquennially_issues_2020	2020-04-12 18:14:53
f6ab29ac-0dd6-4bc4-aa1e-9136bf6c0ecc	41	1	readme_doc	0.8.3	readme_doc	2020-04-12 18:14:53
35a80929-94ca-415a-83fe-abf2dee23c6f	42	1	release_issues	0.8.3	release_issues	2020-04-12 18:14:53
a94f0d86-3a73-470a-8fda-cb9f8239d62d	43	1	requirements_doc	0.8.3	requirements_doc	2020-04-12 18:14:53
6a418106-60b2-4ec9-a2e6-4fb9cec05499	44	1	app_roles	0.8.3	app_roles	2020-04-12 18:14:53
84ec1bac-94fc-43bd-9460-b20a8a89f360	45	1	security_checklist_doc	0.8.3	security_checklist_doc	2020-04-12 18:14:53
b73199d6-9942-433b-97ef-96bddebfae90	46	1	skills	0.8.3	skills	2020-04-12 18:14:53
3f69e913-d970-4b0a-8331-de1c6da38876	47	1	system_guide_doc	0.8.3	system_guide_doc	2020-04-12 18:14:53
efe9717d-ee1e-43d4-a5ee-abca2b8ad7fb	48	1	test_create_table	0.8.3	test_create_table	2020-04-12 18:14:53
140fbe29-60a2-44bd-8565-fc9bdb2e1e93	49	1	test_delete_table	0.8.3	test_delete_table	2020-04-12 18:14:53
dda6fa24-03d4-4ec5-ba5f-45266887db69	50	1	test_empty_table	0.8.3	test_empty_table	2020-04-12 18:14:53
7286578c-e718-4dde-adf5-0bf3c0679ab4	51	1	test_hi_create_table_doc	0.8.3	test_hi_create_table_doc	2020-04-12 18:14:53
6e9f4d86-977d-484b-ab11-0b37f6c3378e	52	1	test_hi_delete_table_doc	0.8.3	test_hi_delete_table_doc	2020-04-12 18:14:53
13eaf174-ed53-4fac-95a4-83c299a7c99e	53	1	test_hierarchy_doc	0.8.3	test_hierarchy_doc	2020-04-12 18:14:53
49f48fb0-b986-4b60-9ec1-b9b89751d083	54	1	test_item	0.8.3	test_item	2020-04-12 18:14:53
21f6a51f-185b-4a25-b8c3-6f02e302a7b5	55	1	test_truncate_table	0.8.3	test_truncate_table	2020-04-12 18:14:53
1ff08076-9d37-4317-a784-e0fb815f3609	56	1	test_update_table	0.8.3	test_update_table	2020-04-12 18:14:53
28dd337a-e57a-4db9-ab3c-8767501867bb	57	1	test_xls_to_db_table	0.8.3	test_xls_to_db_table	2020-04-12 18:14:53
a0be3c42-d2c0-4cd4-b17b-3bc499489f10	58	1	tst_paging	0.8.3	tst_paging	2020-04-12 18:14:53
48cfc41e-ec47-4c42-be7f-e22f40096051	59	1	app_user_roles	0.8.3	app_user_roles	2020-04-12 18:14:53
9fd98df3-711f-4f04-b187-88fe583efabe	61	1	userstories_doc	0.8.3	userstories_doc	2020-04-12 18:14:53
9d837a36-6aa5-42c7-8c4b-ac087c01d371	62	1	yearly_issues	0.8.3	yearly_issues	2020-04-12 18:14:53
d3886a74-83c0-4c1e-803e-5864c8c4d72b	63	1	yearly_issues_2018	0.8.3	yearly_issues_2018	2020-04-12 18:14:53
dc319bf8-8a77-4e4b-8b3a-0cea4fa3eaa4	64	1	yearly_issues_2019	0.8.3	yearly_issues_2019	2020-04-12 18:14:53
356056aa-cd4f-4e62-a61a-d06979a60591	200412203004	1	monthly_issues_202003	0.8.3	monthly_issues_202003	2020-04-12 18:14:53
7ff899c5-d9e2-44cb-99c6-eeb58498a668	30	1	monthly_issues_202001	0.8.3	monthly_issues_202001	2020-04-12 18:14:53
4ef7e5ff-145e-48c2-96fd-c085daa79b2d	200412203024	1	monthly_issues_202004	0.8.3	monthly_issues_202004	2020-04-12 18:14:53
38b4dde8-34d2-4261-92b8-d891b1bfd210	31	1	monthly_issues_202002	0.8.3	monthly_issues_202002	2020-04-12 18:14:53
ca053b66-acc3-4b25-8c53-d3b7ac825fc6	29	1	monthly_issues	0.8.3	monthly_issues	2020-04-12 18:14:53
bbe094a4-2255-45ef-af76-8880ead881c7	65	1	yearly_issues_2020	0.8.3	yearly_issues_2020	2020-04-12 18:14:56
bea09ac2-f835-43d3-9746-d5da2228d4b3	200419113429	1	problems_status	0.8.3	problems_status	2020-04-19 09:07:17
f9fecf76-c485-4cd8-8454-b25f0322d521	60	1	app_users	0.8.3	app_users	2020-04-19 21:08:56
39b3eba3-e262-4e06-872c-d3c5f6085075	200515003827	1	bugs	0.8.5	will hold the bugs in the system	2020-05-14 21:38:46
c6fe1378-0be4-41f5-9662-66983021816a	200511135541	1	app_items_doc	0.8.5	contins the howl menu ...	2020-05-11 10:55:56
c84a7013-19c8-470c-8f2d-c17b7d660fe6	200504204736	2	wow_doc	0.8.4	the way of working doc	2020-05-04 17:48:06
f5b4fd92-039b-46ec-a923-7d1638314373	200509025750	1	incidents	0.8.4	contains the incidents in the qto.fi system	2020-05-08 23:58:20
\.
<file_sep># src/bash/qto/funcs/generate-action-files.help.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doHelpGenerateActionFiles comments ...
# ---------------------------------------------------------
doHelpGenerateActionFiles(){

	do_log "DEBUG START doHelpGenerateActionFiles"
	
	cat doc/txt/qto/helps/generate-action-files.help.txt
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!

	do_log "DEBUG STOP  doHelpGenerateActionFiles"
}
# eof func doHelpGenerateActionFiles


# eof file: src/bash/qto/funcs/generate-action-files.help.sh
<file_sep># src/bash/qto/funcs/create-ctags.help.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doHelpCreateCtags comments ...
# ---------------------------------------------------------
doHelpCreateCtags(){

	do_log "DEBUG START doHelpCreateCtags"
	
	cat doc/txt/qto/helps/create-ctags.help.txt
	sleep 2
	# add your action implementation code here ... 

	do_log "DEBUG STOP  doHelpCreateCtags"
}
# eof func doHelpCreateCtags


# eof file: src/bash/qto/funcs/create-ctags.help.sh
<file_sep>#!/usr/bin/env bash

RUN_UNIT_bash_dir=$(perl -e 'use File::Basename; use Cwd "abs_path"; print dirname(abs_path(@ARGV[0]));' -- "$0")
cd $RUN_UNIT_bash_dir
for i in {1..5} ; do cd .. ; done ; export PRODUCT_DIR=`pwd`;
source $PRODUCT_DIR/.env

# comment out the sudo right from the app user
perl -pi -e 's/^(.*)'"$USER"'(.*$\n)/\#$1'"$USER"'$2/gm' /etc/sudoers

sudo sh /etc/init.d/postgresql start 
bash $PRODUCT_DIR/src/bash/qto/qto.sh -a run-pgsql-scripts


# todo:ysg 
# run the DDL scripts 
# laad the data 

bash $PRODUCT_DIR/src/bash/qto/qto.sh -a mojo-morbo-start
   
while true; do sleep 1000; done;
<file_sep>#  USERSTORIES AND SCENARIOS
* [1. INTRO](#1-intro)
  * [1.1. PURPOSE](#11-purpose)
  * [1.2. AUDIENCE](#12-audience)
  * [1.3. RELATED DOCUMENTATION](#13-related-documentation)
* [2. COMMON UI PERSPECTIVE](#2-common-ui-perspective)
  * [2.1. UI PERFORMANCE](#21-ui-performance)
  * [2.2. FORM FACTORS](#22-form-factors)
    * [2.2.1. Desktop PC form factor UI](#221-desktop-pc-form-factor-ui)
    * [2.2.2. Tablets form factor UI](#222-tablets-form-factor-ui)
    * [2.2.3. Advanced Mobile phones form factor UI](#223-advanced-mobile-phones-form-factor-ui)
  * [2.3. THE OMNI-SEARCH FEATURE](#23-the-omni-search-feature)
    * [2.3.1. List as grid page quick filtering](#231-list-as-grid-page-quick-filtering)
    * [2.3.2. View page quick filtering](#232-view-page-quick-filtering)
    * [2.3.3. Search results page quick filtering](#233-search-results-page-quick-filtering)
  * [2.4. THE LEFT MENU UI - ORGANISING THE STRUCTURE OF THE PROJECT](#24-the-left-menu-ui--organising-the-structure-of-the-project)
    * [2.4.1. Opening and closing of the left menu](#241-opening-and-closing-of-the-left-menu)
    * [2.4.2. Managing listings from the left menu](#242-managing-listings-from-the-left-menu)
    * [2.4.3. Managing documents from the left menu](#243-managing-documents-from-the-left-menu)
    * [2.4.4. Managing folders on the left menu](#244-managing-folders-on-the-left-menu)
    * [2.4.5. Fixing the left menu](#245-fixing-the-left-menu)
    * [2.4.6. Drag to widen or shrink the left menu](#246-drag-to-widen-or-shrink-the-left-menu)
  * [2.5. THE LIST PAGE UI](#25-the-list-page-ui)
    * [2.5.1. Search in the list page](#251-search-in-the-list-page)
    * [2.5.2. Items listing](#252-items-listing)
    * [2.5.3. Items editing](#253-items-editing)
      * [2.5.3.1. Items editing inline](#2531-items-editing-inline)
      * [2.5.3.2. Items editing via a modal dialog](#2532-items-editing-via-a-modal-dialog)
      * [2.5.3.3. Items editing via dropbox](#2533-items-editing-via-dropbox)
      * [2.5.3.4. Items editing via calendar control](#2534-items-editing-via-calendar-control)
      * [2.5.3.5. Commenting on items](#2535-commenting-on-items)
    * [2.5.4. Items creation](#254-items-creation)
    * [2.5.5. Items deletion](#255-items-deletion)
    * [2.5.6. Items export](#256-items-export)
    * [2.5.7. Items import](#257-items-import)
  * [2.6. THE VIEW DOC PAGE UI](#26-the-view-doc-page-ui)
    * [2.6.1. View page navigation](#261-view-page-navigation)
      * [2.6.1.1. Title numbers link navigation and sharing](#2611-title-numbers-link-navigation-and-sharing)
      * [2.6.1.2. Title numbers link navigation and sharing](#2612-title-numbers-link-navigation-and-sharing)
      * [2.6.1.3. Access items by theirs ids](#2613-access-items-by-theirs-ids)
      * [2.6.1.4. Keyboard navigation and editing](#2614-keyboard-navigation-and-editing)
          * [2.6.1.4.1. Tabbing direction flow of tabbing](#26141-tabbing-direction-flow-of-tabbing)
          * [2.6.1.4.2. TOC keyboard navigation](#26142-toc-keyboard-navigation)
          * [2.6.1.4.3. Faster text editing when tabbing](#26143-faster-text-editing-when-tabbing)
    * [2.6.2. View doc page print preview](#262-view-doc-page-print-preview)
      * [2.6.2.1. Disable editing in print preview mode but keep the links](#2621-disable-editing-in-print-preview-mode-but-keep-the-links)
      * [2.6.2.2. Disable the right and left menus in print preview mode](#2622-disable-the-right-and-left-menus-in-print-preview-mode)
    * [2.6.3. Quick search in the view page ](#263-quick-search-in-the-view-page-)
    * [2.6.4. Hierarchical items management](#264-hierarchical-items-management)
      * [2.6.4.1. Add Item](#2641-add-item)
      * [2.6.4.2. Update item](#2642-update-item)
          * [2.6.4.2.1. Update item's title](#26421-update-item's-title)
          * [2.6.4.2.2. Update item's description](#26422-update-item's-description)
              * [2.6.4.2.2.1. Add interlinking in description](#264221-add-interlinking-in-description)
          * [2.6.4.2.3. Update item's src_code](#26423-update-item's-src_code)
      * [2.6.4.3. Delete item](#2643-delete-item)
    * [2.6.5. Move item in the hierarchy by drag and drop](#265-move-item-in-the-hierarchy-by-drag-and-drop)
      * [2.6.5.1. Move item in the hierarchy](#2651-move-item-in-the-hierarchy)
      * [2.6.5.2. Search in the right menu](#2652-search-in-the-right-menu)
      * [2.6.5.3. Links in the right menu](#2653-links-in-the-right-menu)
    * [2.6.6. Left menu navigation](#266-left-menu-navigation)
    * [2.6.7. Hierarchy management](#267-hierarchy-management)
      * [2.6.7.1. Single item move in the hierarchy ](#2671-single-item-move-in-the-hierarchy-)
      * [2.6.7.2. Single item move in the hierarchy ](#2672-single-item-move-in-the-hierarchy-)
    * [2.6.8. Pictures management in the view doc page](#268-pictures-management-in-the-view-doc-page)
      * [2.6.8.1. Add picture](#2681-add-picture)
      * [2.6.8.2. Edit picture](#2682-edit-picture)
          * [2.6.8.2.1. Resize picture](#26821-resize-picture)
          * [2.6.8.2.2. Move picture left and right in the item's paragraph](#26822-move-picture-left-and-right-in-the-item's-paragraph)
      * [2.6.8.3. Delete picture](#2683-delete-picture)
    * [2.6.9. Multimedia links management](#269-multimedia-links-management)
      * [2.6.9.1. Edit embedded multimedia link](#2691-edit-embedded-multimedia-link)
      * [2.6.9.2. Edit embedded multimedia link](#2692-edit-embedded-multimedia-link)
      * [2.6.9.3. Delete embedded multimedia link](#2693-delete-embedded-multimedia-link)
  * [2.7. THE GLOBAL SEARCH PAGE / THE SEARCH UI](#27-the-global-search-page-/-the-search-ui)
    * [2.7.1. Global search from the list page](#271-global-search-from-the-list-page)
    * [2.7.2. Global search from the view page](#272-global-search-from-the-view-page)
    * [2.7.3. Global search modal dialog ui](#273-global-search-modal-dialog-ui)
      * [2.7.3.1. Search-autocomplete on modal search dialog](#2731-search-autocomplete-on-modal-search-dialog)
  * [2.8. THE PRESS PAGE](#28-the-press-page)
    * [2.8.1. URL naming convention of the press page](#281-url-naming-convention-of-the-press-page)
    * [2.8.2. Articles listing from the press page DAILY](#282-articles-listing-from-the-press-page-daily)
    * [2.8.3. Articles listing from the press page MONTHLY](#283-articles-listing-from-the-press-page-monthly)
    * [2.8.4. Articles listing from the press page MONTHLY](#284-articles-listing-from-the-press-page-monthly)
    * [2.8.5. Accessing a single article / blog post by url](#285-accessing-a-single-article-/-blog-post-by-url)
  * [2.9. THE REPORT PAGE](#29-the-report-page)
  * [2.10. THE NOTES PAGE / UI](#210-the-notes-page-/-ui)
  * [2.11. THE CHAT UI](#211-the-chat-ui)
  * [2.12. MULTI-PROJECT AWARE UI](#212-multi-project-aware-ui)
    * [2.12.1. Items data transfer between different projects](#2121-items-data-transfer-between-different-projects)
    * [2.12.2. Items move](#2122-items-move)
  * [2.13. ISSUES IMPORT FROM GOOGLE CALENDAR](#213-issues-import-from-google-calendar)
* [3. TEAM LEADER PERSPECTIVE](#3-team-leader-perspective)
  * [3.1. SECURITY](#31-security)
    * [3.1.1. Data security](#311-data-security)
    * [3.1.2. Roles management](#312-roles-management)
    * [3.1.3. Users management](#313-users-management)
      * [3.1.3.1. Users management in a project](#3131-users-management-in-a-project)
      * [3.1.3.2. Users management in a project](#3132-users-management-in-a-project)
      * [3.1.3.3. Personal data handling minimisation](#3133-personal-data-handling-minimisation)
  * [3.2. PROJECTS MANAGEMENT](#32-projects-management)
    * [3.2.1. Grant access per project](#321-grant-access-per-project)
  * [3.3. SECURE ACCESS TO DATA](#33-secure-access-to-data)
    * [3.3.1. Role based secure access to data per table or report for role](#331-role-based-secure-access-to-data-per-table-or-report-for-role)
    * [3.3.2. Role based secure access to data per row](#332-role-based-secure-access-to-data-per-row)
  * [3.4. TIME MANAGEMENT](#34-time-management)
    * [3.4.1. Closing of a period](#341-closing-of-a-period)
    * [3.4.2. Use monthly_issues status report](#342-use-monthly_issues-status-report)
* [4. TEAM MEMBER BIZ PERSPECTIVE](#4-team-member-biz-perspective)
  * [4.1. ISSUES MANAGEMENT](#41-issues-management)
    * [4.1.1. Remove existing issues](#411-remove-existing-issues)
    * [4.1.2. Search for existing issues](#412-search-for-existing-issues)
    * [4.1.3. Track issues progress](#413-track-issues-progress)
    * [4.1.4. Track issues history](#414-track-issues-history)
  * [4.2. ITEMS MANAGEMENT](#42-items-management)
  * [4.3. ISSUES MANAGEMENT](#43-issues-management)
    * [4.3.1. Update existing Items](#431-update-existing-items)
    * [4.3.2. Remove existing Items](#432-remove-existing-items)
    * [4.3.3. Search for existing Items](#433-search-for-existing-items)
    * [4.3.4. Track Items progress](#434-track-items-progress)
    * [4.3.5. Track Items history](#435-track-items-history)
  * [4.4. TRACK ISSUES RELATIONS](#44-track-issues-relations)
  * [4.5. ITEMS MANAGEMENT](#45-items-management)
  * [4.6. MEASURE SUCCESS](#46-measure-success)
  * [4.7. MONITOR SUCCESS](#47-monitor-success)
  * [4.8. TRACK ISSUES RELATIONS](#48-track-issues-relations)
  * [4.9. TIME MANAGEMENT](#49-time-management)
    * [4.9.1. time centric planning](#491-time-centric-planning)
    * [4.9.2. time centric reporting](#492-time-centric-reporting)
  * [4.10. PROJECT'S PERSONS ISSUE COMBINATIONS](#410-project's-persons-issue-combinations)
* [5. PROJECT OBSERVER BIZ PERSPECTIVE](#5-project-observer-biz-perspective)
  * [5.1. PROJECTS OBSERVATION](#51-projects-observation)
  * [5.2. MEASURE SUCCESS](#52-measure-success)
  * [5.3. ISSUES OBSERVATION](#53-issues-observation)
  * [5.4. MONITOR SUCCESS](#54-monitor-success)
* [6. PRODUCT INSTANCE OWNER PERSPECTIVE](#6-product-instance-owner-perspective)
  * [6.1. SYSTEM DEPLOYABILITY](#61-system-deployability)
  * [6.2. TIME MANAGEMENT](#62-time-management)
  * [6.3. SYSTEM STABILITY](#63-system-stability)
  * [6.4. GENERIC CRUDS FOR ITEMS](#64-generic-cruds-for-items)
  * [6.5. SYSTEM RELIABILITY](#65-system-reliability)
  * [6.6. PROJECT'S PERSONS ISSUE COMBINATIONS](#66-project's-persons-issue-combinations)
  * [6.7. EASE OF USE](#67-ease-of-use)
  * [6.8. SECURITY](#68-security)
    * [6.8.1. Password strength security](#681-password-strength-security)
    * [6.8.2. Password reset enforceability](#682-password-reset-enforceability)
    * [6.8.3. Login info](#683-login-info)
* [7. PROJECT OBSERVER BIZ PERSPECTIVE](#7-project-observer-biz-perspective)
  * [7.1. PROJECTS OBSERVATION](#71-projects-observation)
    * [7.1.1. Who logged in and out when](#711-who-logged-in-and-out-when)
  * [7.2. ISSUES OBSERVATION](#72-issues-observation)
    * [7.2.1. Who changed what and why](#721-who-changed-what-and-why)
* [8. ETL AND INTEGRATIONS PERSPECTIVE](#8-etl-and-integrations-perspective)
  * [8.1. DATABASE TO JSON FILES DATA LOAD](#81-database-to-json-files-data-load)
  * [8.2. DATABASE METADATA EXPORT](#82-database-metadata-export)
  * [8.3. SYSTEM PERFORMANCE](#83-system-performance)
  * [8.4. JSON FILES TO DB DATA LOAD](#84-json-files-to-db-data-load)
  * [8.5. XLS-TO-POSTGRES-DB HIERARCHICAL DATA LOAD](#85-xls-to-postgres-db-hierarchical-data-load)
    * [8.5.1. error reporting in xls-to-postgres-db hierarchical data load ](#851-error-reporting-in-xls-to-postgres-db-hierarchical-data-load-)
  * [8.6. PROJECT DEPLOYMENT](#86-project-deployment)
* [9. DEVOPS ENGINEER PERSPECTIVE](#9-devops-engineer-perspective)
  * [9.1. DOCUMENTATION](#91-documentation)
  * [9.2. ETL](#92-etl)
    * [9.2.1. Quick start documentation](#921-quick-start-documentation)
    * [9.2.2. Application's source code and documentation integrity](#922-application's-source-code-and-documentation-integrity)
  * [9.3. INSTALLATION](#93-installation)
    * [9.3.1. A single shell call installation](#931-a-single-shell-call-installation)
    * [9.3.2. A reliable installation documentation](#932-a-reliable-installation-documentation)
  * [9.4. TESTING](#94-testing)
    * [9.4.1. Clarity and brevity of the end to end tests](#941-clarity-and-brevity-of-the-end-to-end-tests)
    * [9.4.2. Databases - local to remote and remote to local sync](#942-databases--local-to-remote-and-remote-to-local-sync)
    * [9.4.3. Abort end-to-end tests on single test fail](#943-abort-end-to-end-tests-on-single-test-fail)
    * [9.4.4. Single shell call for e2e testing](#944-single-shell-call-for-e2e-testing)
  * [9.5. ETL](#95-etl)
    * [9.5.1. Database to json files data load](#951-database-to-json-files-data-load)
    * [9.5.2. Items data transformations](#952-items-data-transformations)
    * [9.5.3. Json files to db data load](#953-json-files-to-db-data-load)
    * [9.5.4. Xls-to-postgres-db hierarchical data load](#954-xls-to-postgres-db-hierarchical-data-load)
    * [9.5.5. Log entries format configuration](#955-log-entries-format-configuration)
    * [9.5.6. Tool run log to human readable description](#956-tool-run-log-to-human-readable-description)
    * [9.5.7. Components start run message print](#957-components-start-run-message-print)
    * [9.5.8. Tool exit with exit code and exit message](#958-tool-exit-with-exit-code-and-exit-message)
  * [9.6. RUN-TIME MANAGEMENT](#96-run-time-management)
    * [9.6.1. Logging](#961-logging)
    * [9.6.2. Single shell call for projects switching](#962-single-shell-call-for-projects-switching)
    * [9.6.3. Start and Stop of the Application](#963-start-and-stop-of-the-application)
  * [9.7. SECURITY](#97-security)
    * [9.7.1. Security modes](#971-security-modes)
* [10. UI DEVELOPER PERSPECTIVE](#10-ui-developer-perspective)
  * [10.1. FOREIGN KEYS (FK) CONFIGURABILITY](#101-foreign-keys-(fk)-configurability)
  * [10.2. CODE TRACEABILITY BY UUID](#102-code-traceability-by-uuid)
  * [10.3. SINGLE SHELL CALL FOR PROJECTS SWITCHING](#103-single-shell-call-for-projects-switching)
  * [10.4. ISSUES PUBLISHING FROM SHELL CALLS](#104-issues-publishing-from-shell-calls)
  * [10.5. METADATA HANDLING](#105-metadata-handling)
  * [10.6. EASY SETUP FOR TESTABILITY](#106-easy-setup-for-testability)




    

## 1. INTRO


    

### 1.1. Purpose
The purpose of this document is to present the userstories  in the qto application for the version of this instance. Simply stated this is the document, in which any user gets to express their wishes towards the qto application, but in structurized  manner.

    

### 1.2. Audience
This document is aimed for any potential and actual users of the qto application. Product Owners, Developers and Architects working on the application MUST read and understand this document at least to the extend of their own contribution for the application. 

    

### 1.3. Related documentation
This document is part of the QTO application documentation-set, which contains the following documents:
 - readme_doc-0 - the initial landing readme doc for the project
 - userstories_doc-0 - the collection of user-stories used to describe "what is desired"
 - requirements_doc-0 - the structured collection of the requirements 
 - system_guide_doc-0 - architecture and System description
 - devops_guide_doc-0 - a guide for the developers and devops operators
 - installations_doc-0 - a guide for installation of the application
 - enduser_guide_doc-0 - the guide for the usage of the UI ( mainly ) for the end-users
 - concepts_doc-0 - the concepts doc 

You can access all the latest  qto documentation from qto site: 
https://qto.fi
in it's native format.
All the documents are updated and redistributed in combination of the current version of the application in both md and pdf file format and can be found under the following directories:
 - doc/md
 - doc/pdf

    

## 2. COMMON UI PERSPECTIVE
As an UI user of the qto application 
In order to manage my issues via the UI successfully  
I want to have a nice user experience while using the qto application.

    

### 2.1. UI Performance
As an UI user of the qto application 
In order to enjoy the usage of the tool and interact efficiently   
I wan to to  have responsive and quick UI.

    

### 2.2. Form factors
As an UI user of the qto application 
In order to be able to be able to access from anywhere and anytime
I want to be able to use it with different form factor devices

    

#### 2.2.1. Desktop PC form factor UI
As an UI user of the qto application 
In order to be able to use it fully on the work-place
I want to it to work primarily on Desktop for every UI feature

    

#### 2.2.2. Tablets form factor UI
As an UI user of the qto application 
In order to be able to access it quickly on the go 
I want to  be able to use the same UI on an advanced tablets

    

#### 2.2.3. Advanced Mobile phones form factor UI
As an UI user of the qto application 
In order to be able to access it quickly on the go
I want to  be able to use the same UI on an advanced mobile phones. 

    

### 2.3. The Omni-search feature
As an UI user of the qto application
In order to have a consistent experience around all the user interfaces for search and get quickly to the information on EACH page and/or each table of the database
I want to be able to perform quick filtering on ALL the UI elements from the page from a single omni search-box placed at the top bar of the screen.

    

#### 2.3.1. List as grid page quick filtering
As an UI user of the qto application
In order to get quickly to the information on the listing as grid page
I want to be able to perform quick filtering on both the grid and the left menu.

    

#### 2.3.2. View page quick filtering
As an UI user of the qto application
In order to get quickly to the information on the view page
I want to be able to perform quick filtering on both the information on the view page and the left menu.

    

#### 2.3.3. Search results page quick filtering
As an UI user of the qto application
In order to get quickly to the information on the search as grid page
I want to be able to perform quick filtering on both the grid and the left and right menu.

    

### 2.4. The left menu UI - organising the structure of the project
As an UI user of the qto application
in order to be able to manage the whole structure of my project
I want to have access to a left menu from each page.

    

#### 2.4.1. Opening and closing of the left menu
As an UI user of the qto application
in order to be able to quickly get to different pages of the project
I want to be able to open the left menu by clicking on the top menu button and close it by clicking anywhere else in the document.

    

#### 2.4.2. Managing listings from the left menu
As an UI user of the qto application
in order to be able to store and track data from different listings or in different documents 
I want to be able to create new listings by simple right-click on the left menu and selecting "add listing".

    

#### 2.4.3. Managing documents from the left menu
As an UI user of the qto application
in order to be able to store and track data from different listings or in different documents 
I want to be able to create new documents by simple right-click on the right menu and selecting "add document".

    

#### 2.4.4. Managing folders on the left menu
As an UI user of the qto application
in order to be able to be able to organise the structure of my project
I want to be able to add, update and delete folders from the left menu

    

#### 2.4.5. Fixing the left menu
As an UI user of the qto application
in order to be able to be able to better manipulate the items on the left menu
I want to be able to fix it to not close when clicking somewhere else ....

    

#### 2.4.6. Drag to widen or shrink the left menu
As an UI user of the qto application
in order to be able to be able to better manipulate the items on the left menu
I want to be able to drag the border of the left menu to either widen it or to shrink it , so that a closing button would be visible on left menu hover.

    

### 2.5. The list page UI
As an UI user of the qto application 
in order to be able to efficiently process relational data for any project 
I want to be able to list all or part of the project's database table via a single UI

    

#### 2.5.1. Search in the list page
As an UI user of the qto application
In order to be able to quickly and effortlessly search the content of the loaded document in the list page
I want to be able to focus the search-box with a single shortcut / click on the left menu, so that after typing 3 letters the left menu will get filtered by the search string and the table content will get filtered as well.

    

#### 2.5.2. Items listing
As an qto ui user 
In order to be able to quickly see as much items ( issues, problems, ideas etc. ) 
I want to list the items in a web page according to the filtering criteria I might have specified earlier on … 

    

#### 2.5.3. Items editing
aAs an UI user of the qto application 
In order to update the application data via the UI 
I wan to to be able to edit the data for ANY of the items in the application I have access to. 

    

##### 2.5.3.1. Items editing inline
As an UI user of the qto application 
In order to be able to quickly update the items data in a listing UI
I wan to to be able to edit it Excel table like by quickly navigating trough an grid and type the new values to update.

    

##### 2.5.3.2. Items editing via a modal dialog
As an UI user of the qto application 
In order to be able update the application data via the UI by using a more familiar form-like interface
I want to to be able to edit the data for any item by clicking on an edit button, and filling the fields of a form

    

##### 2.5.3.3. Items editing via dropbox
As an UI user of the qto application 
In order to be able to update foreign key tables data via the UI 
I want to to be able to edit the data for any item by : 
 - clicking on an a dropbox and either choosing or searching and choosing the human readable value

    

##### 2.5.3.4. Items editing via calendar control
As an UI user of the qto application 
In order to be able to update date-time data in the listing page via the UI
I want to to be able to edit the data for any item by : 
 - clicking on an a calendar control and choosing the date and the time from it

    

##### 2.5.3.5. Commenting on items
As an UI user of the qto application 
In order to be able to have item centric discussion
I want to to be able to open a comments chat like section at the right of the list page screen, which will pre-pend comments in a chat like fassion

    

#### 2.5.4. Items creation
As an UI user of the application 
I order to create new items in the application 
I want to be able to create them via the UI 
by clicking "create new button" and filling as few as possible data entries and clicking a Save button for ANY of the items in the application. 

    

#### 2.5.5. Items deletion
As an UI user of the application 
I order to delete existing items in the project
I want to  be able to delete them via the UI 
by clicking a "delete " button and confirming the deletion for the item

    

#### 2.5.6. Items export
As an UI user of the application 
I order to export the data  of the items in the application 
I want to be able to perform the following data export from a single button click on the UI or a single url access:
 - export to md docs ( github , azure )
 - export to pdf docs
 - export to xls files
 - export to msft docx docs
 - export to json dump files 

    

#### 2.5.7. Items import
As an UI user of the application 
I order to import data  of the items in the application 
I want to  be able to perform the following data imports in the UI from a single button click and pointing to the file path of a file
 - import from md docs
 - import from pdf docs
 - import from xls files
 - import from msft docx docs
 - import from json dump files 

    

### 2.6. The view doc page UI
As an UI user of the qto application 
in order to be able to efficiently process hierarchical relational data for any project 
I want to be able to list all or part of the project's database table via a single UI

    

#### 2.6.1. View page navigation
As an UI user 
In order to quickly grasp the content of a view doc 
I want to be able to quickly and effortlessly navigate the view doc page.

    

##### 2.6.1.1. Title numbers link navigation and sharing
As an UI user 
In order to read the the current item in the top of the screen and to be able to refer it exactly with a link 
I want to be able on click the system to move it on the top of the screen and to change the url pointing to the exactly chosen item.

    

##### 2.6.1.2. Title numbers link navigation and sharing
As an UI user 
In order to read the the current item in the top of the screen and to be able to refer it exactly with a link 
I want to be able on click the system to move it on the top of the screen and to change the url so that when the link is copied one could refer to it by sending it.

    

##### 2.6.1.3. Access items by theirs ids
As an UI user
In order to be able to quickly access the content of a particular item
I want the System to be capable to scroll to it on the top of the view port of the page after I have clicked a link to it.

    

##### 2.6.1.4. Keyboard navigation and editing
As an UI user
In order to be able to quickly navigate throughout the whole page once loaded
I want to be able to cycle every manageable element of the page with the keyboard in a logical order from the top till the bottom

    

###### 2.6.1.4.1. Tabbing direction flow of tabbing
As an UI user
In order to be able to quickly navigate throughout the whole page once loaded
I want the cycling order to go from the omni-search box to the right-toc menu than to the titles and paragraphs of the document

    

###### 2.6.1.4.2. TOC keyboard navigation
As an UI user
In order to be able to quickly navigate throughout the Table Of Contents section on the right page menu
I want to be able to cycle every manageable element in the TOC menu with the keyboard in a logical order from the top till the bottom and get visual indication for the selected / active links from the UI.

    

###### 2.6.1.4.3. Faster text editing when tabbing
As an UI user
In order to be able to quickly delete text from the titles and/or the paragraphs
I want the UI to select the whole text of the title and the paragraphs while active, so that I could quickly copy or remove the text from the keyboard.

    

#### 2.6.2. View doc page print preview
As an UI user
In order to be able to share the documents of a qto application in pdf format or even physically print them
I want to be able to see the whole document or just a branch of it in print-preview mode

    

##### 2.6.2.1. Disable editing in print preview mode but keep the links
As an UI user 
In order to have as clean entity as possible to a pdf document I do not want to be able to add,edit or delete items from the print preview mode, however so that the clickable links should remain as part of the content

    

##### 2.6.2.2. Disable the right and left menus in print preview mode
As an UI User
In order to have as clean and easily printable document as possible in a pdf format I do not want to either see or access the right and left menu links.

    

#### 2.6.3. Quick search in the view page 
As an UI user of the qto application
In order to be able to quickly and effortlessly search the content of the loaded document in the view doc page
I want to be able to focus the search-box with a single shortcut / click and the document to be filtered by the content of the input search box, by highlighting the search item.

    

#### 2.6.4. Hierarchical items management
As an the ui user
In order to manage the hierarchical items in the application
I want to be able to manage by actions (list, create, update, delete, search) ANY hierarchical items of the application in from a hierarchical doc format by simply right-clicking on their titles and choosing from the menut the action which must have both image and text to indicate more 

    

##### 2.6.4.1. Add Item
I want to be able to add items to the view page by simply right-clicking the upper title of the item and selecting add item

    

##### 2.6.4.2. Update item
I want to be able to update items data to the view page

    

###### 2.6.4.2.1. Update item's title
I want to be able to update item's title by simply clicking in it and starting to type the new contents ...

    

###### 2.6.4.2.2. Update item's description
I want to be able to update item's descrption by simply clicking in it and starting to type the new contents ...

    

####### 2.6.4.2.2.1. Add interlinking in description


    

###### 2.6.4.2.3. Update item's src_code
I want to be able to update item's src_code by simply clicking in it and starting to type the new contents ...

    

##### 2.6.4.3. Delete item
As an UI user of the qto application
In order to quickly edit the structure of a hierarchical doc 
I want to be able to move item by drag and drop from the left menu.

    

#### 2.6.5. Move item in the hierarchy by drag and drop
As an the ui user
In order to navigate quickly in the document structure
I want to be able to see the document structure by clicking a right menu containing the Table of Contents of this document with a clickable links.


    

##### 2.6.5.1. Move item in the hierarchy
As an UI user of the qto application
In order to quickly edit the structure of a hierarchical doc 
I want to be able to move item by drag and drop from the left menu.

    

##### 2.6.5.2. Search in the right menu
As an the ui user
In order to navigate quickly in the document structure
I want to be able filter quickly the right menu items by the means of right menu quick srch/filtering box which will filter the items containing the string I am typing for dynamically

    

##### 2.6.5.3. Links in the right menu
As an the ui user
In order to navigate quickly in the document structure by levels
I want to be able to simply click on the links ( level 1 , level-2 , level-3 etc. ) which will open the document

    

#### 2.6.6. Left menu navigation
As an UI user of the qto application 
In order to be able to quickly jump to a different document / listing 
I want to be able to click on the upper left corner of the view page and access the hierarchical project structure of the whole project from the left menu.

    

#### 2.6.7. Hierarchy management
As an UI user of the qto application 
In order to be able to quickly jump to a different document / listing 
I want to be able to click on the upper left corner of the view page and access the hierarchical project structure of the whole project from the left menu.

    

##### 2.6.7.1. Single item move in the hierarchy 
As a qto application UI user 
In order to be able to move a single item anywhere in the hierarchy
I want to be able to move it by drag and drop on the left side of the document.

    

##### 2.6.7.2. Single item move in the hierarchy 
As a qto application UI user 
In order to be able to move a single item and all it's descendants anywhere in the hierarchy
I want to be able to move it by drag and drop on the left side of the document.

    

#### 2.6.8. Pictures management in the view doc page


    

##### 2.6.8.1. Add picture
As a qto application UI user 
In order to have pictures in the documents
I want to be able to add pictures by ...

    

##### 2.6.8.2. Edit picture
As a qto application UI user 
In order to have pictures in the documents
I want to be able to add pictures by ...

    

###### 2.6.8.2.1. Resize picture
As a qto application UI user 
In order to have a better visualisation of the pictures in the document
I want to be able to resize the pictures  pictures by simply dragging their corners

    

###### 2.6.8.2.2. Move picture left and right in the item's paragraph
As a qto application UI user 
In order to have a better visualisation of the pictures in the document
I want to be able to move the  pictures by simply dragging them left or right.

    

##### 2.6.8.3. Delete picture
As a qto application UI user 
In order to have pictures in the documents
I want to be able to delete pictures by ...

    

#### 2.6.9. Multimedia links management


    

##### 2.6.9.1. Edit embedded multimedia link


    

##### 2.6.9.2. Edit embedded multimedia link


    

##### 2.6.9.3. Delete embedded multimedia link


    

### 2.7. The global search page / the search UI
As an qto ui user 
In order to be able to search ANY items ( issues, problems, ideas etc. ) 
I want to have a pop-up search SearchBox with dimmed background providing with interactive autocomplete, which would assist me in specifying the search criteria for any item I want to list. 

    

#### 2.7.1. Global search from the list page
As an qto ui user 
In order to be able to search ANY items ( issues, problems, ideas etc. ) from the list page
I want to be able to search from the list page by simply clicking on a search textbox on the top left of the page, typing the search phrase I want to search for and getting the results by hitting enter or clicking the search icon on the right of the search box ...

    

#### 2.7.2. Global search from the view page
As an qto ui user 
In order to be able to search ANY items ( issues, problems, ideas etc. ) from the view page
I want to be able to search from the view page by simply clicking on a search icon ( might be the / shortcut as well ) , the system should bring up a modal dialog with the global search ui, than I should be able to type the search phrase I want to search for and hit enter, when the System should present me the search results.

    

#### 2.7.3. Global search modal dialog ui
As an UI user 
In order to have the best possible search experience
I want to be able to search from a modal dialog UI which will pop-up on a dimmed background after the search-box is focused .. 

    

##### 2.7.3.1. Search-autocomplete on modal search dialog
As an UI user 
In order to get quickly to exactly the searched item I am searching for
I want the System to present me with autocomplete of the mostly occurring string I am searching for with the name of the item on the left and when clicking on it I want it to open the list as grid page with all the items having this item in the search ... 

    

### 2.8. The press page


    

#### 2.8.1. URL naming convention of the press page
As the reader of any qto instance 
In order to be able to receive and send short URLs from the articles collection of this instance
I want to have a short and time based url in order to be able to quickly access it from the history of my browser.

    

#### 2.8.2. Articles listing from the press page DAILY
As the reader of a qto instance
In order to be able to quickly access all the articles per DAY
I want to access all the articles per day by accessing the following url : 
&lt;&lt;proj&gt;&gt;/press/&lt;&lt;yyyy&gt;&gt;-&lt;&lt;mm&gt;&gt;-&lt;&lt;dd&gt;&gt;

    

#### 2.8.3. Articles listing from the press page MONTHLY
As the reader of a qto instance
In order to be able to quickly access all the articles per MONTH
I want to access all the articles per day by accessing the following url : 
&lt;&lt;proj&gt;&gt;/press/&lt;&lt;yyyy&gt;&gt;-&lt;&lt;mm&gt;&gt;

    

#### 2.8.4. Articles listing from the press page MONTHLY
As the reader of a qto instance
In order to be able to quickly access all the articles per YEAR
I want to access all the articles per day by accessing the following url : 
&lt;&lt;proj&gt;&gt;/press/&lt;&lt;yyyy&gt;&gt;

    

#### 2.8.5. Accessing a single article / blog post by url
As the reader of a qto instance
In order to be able to quickly access a single article
I want to access a single article by using the following naming convention
&lt;&lt;proj&gt;&gt;/press/&lt;&lt;yyyy&gt;&gt;-&lt;&lt;mm&gt;&gt;-&lt;&lt;dd&gt;&gt;/&lt;&lt;article-short-title&gt;&gt;

    

### 2.9. The report page
As the UI of the qto application
In order to be able to quickly view the results of predefined reports ( which are actually stored_procedures in the product databases ) 
I want to be able to execute a report by providing the following syntax

     <<server>>:<<port>>/<<proj-db>>/report/<<stored-procedure-name>>?p_01=<<parameter-01-value>>&?p_02=<<parameter-02-value>> ... 

### 2.10. The notes page / UI
The 7 sticky notes would be used for practising :
 - the cruds with web sockets
 - the per user data management

    

### 2.11. The chat UI
As UI user
In order to be able to quickly communicate on anything with other qto users of the qto applications 
I want to be able to use a quick chat with other users within the application instance

    

### 2.12. Multi-project aware UI
As the UI user of an qto instance 
In order to save be able to track my personal time usage between different projects and the different interdependencies 
I want to be able to move items data from one project to another via the UI.

    

#### 2.12.1. Items data transfer between different projects
As the UI user of an qto instance 
In order to manage multiple projects efficiently
I want to be able to move items data from one project to another via the UI in a trusted manner.

    

#### 2.12.2. Items move
As an UI user of the application 
I order to move the items into different tables
I want to  be able to move them from a button in the listing page by specifying the target table

    

### 2.13. Issues import from Google calendar
As the UI user of an qto instance 
In order to be able to visualise and manage my  start- and stop_time having issues better 
I want to  be able to import my Google calendar issues into my qto profile on an qto instance

    

## 3. TEAM LEADER PERSPECTIVE
As a team leader
In order to operate successfully one or many projects of my team(s)  
I want to have a nice user experience while using the qto tool.

    

### 3.1. Security


    

#### 3.1.1. Data security
As the biz team leader In order to protect the data of the project, which is vital for its existence \nI don't want any person but me to have full visibility and control of the data in a project.

    

#### 3.1.2. Roles management
As the Instance Owner or Product Owner 
in order to have a finer granularity on the different data stored in my instance 
I want to be able to define app_roles and per app_roles visibility on : 
 - item page level 
 - item row level
 - item tag level

    

#### 3.1.3. Users management
As a team leader 
In order provide the persons and programs access to my project
I want to provide read, write access to the data and execute access ( run DDL's) per table

    

##### 3.1.3.1. Users management in a project
As an team leader
In order to be able to efficiently allocate human resources to a project
I want to be able manage ( create , update , delete and search ) users for each project.

    

##### 3.1.3.2. Users management in a project
As an team leader
In order to be able to efficiently allocate human resources to a project
I want to be able manage ( create , update , delete and search ) users for each project.

    

##### 3.1.3.3. Personal data handling minimisation
As a team leader 
In order to avoid legal obligations and complex procedures, while handling personal data
I want to be able to handle the interpersonal exchange of data by collecting ONLY the e-mail of the persons or programs participating in the project.

    

### 3.2. Projects management
As an team leader
In order to be able to manage multiple projects 
I want to  be able to create , update and remove projects.

    

#### 3.2.1. Grant access per project
As a team leader 
In order to enrol authenticated users into the project I am responsible to 
I want to be able to grant them with access by only writing their e-mail into a text field and clicking invite button. 

    

### 3.3. Secure access to data
As the team leader of a qto project
In order 
I want to be able to grant separate read, write and execute access per table and role

    

#### 3.3.1. Role based secure access to data per table or report for role
As the team leader of a qto project
In order to restrict the certain operations to certain tables for the authenticated app_roles
I want to be able to grant separate read, write and execute access per table and role.

    

#### 3.3.2. Role based secure access to data per row
As the team leader of a qto project
In order 
I want to be able to grant separate read, write and execute access per table and role

    

### 3.4. Time management
As an team leader
In order to be able the maximise the performance of the team for qto used periods 
I want to to be able to manage time efficiently 
by accessing a simple page containing its value and the period it is related to.

    

#### 3.4.1. Closing of a period
As the team leader of a qto project
In order to keep track the history of 
I want to to be able to manage time efficiently 
by accessing a simple page containing its value and the period it is related to.

    

#### 3.4.2. Use monthly_issues status report
As the team leader of a qto project
In order to keep track on the statuses of the ongoing issues in my project 
I want to to be able to accesss and read a visual report containing pie chart with the amount of the issues at any moment and their distributions by status

    

## 4. TEAM MEMBER BIZ PERSPECTIVE
As a team member
In order to operate successfully in the project  
I want to have a nice user experience while using the qto application
by being able to manage all the items in the application ( issues,questions,problems , etc. ) 

    

### 4.1. Issues management
As a team member of the qto
In order to achieve the best possible efficiency during the work on one or many projects
I want to be able to manage the issues in those projects. 

    

#### 4.1.1. Remove existing issues
As an team member
In order to be able to stop the work on existing issues
I want to be able to remove issues via the qto

    

#### 4.1.2. Search for existing issues
As an team member
In order to be able to change attributes of the issues I am responsible for
I want to be able to update the issues' data. 

    

#### 4.1.3. Track issues progress
As an team member
In order to be able to quickly access existing issues
I want to be able to search the issues. 

    

#### 4.1.4. Track issues history
As a team member
In order to keep track on the changes of the issues
I want to be able to follow the history of the changes


    

### 4.2. Items management
As a team member of the qto
In order to achieve the best posible efficiency during the work on one or many projects
I want to be able to manage the Items in those projects, where items could be ( problems , questions etc.

    

### 4.3. Issues management
As a team member of the qto
In order to achieve the best possible efficiency during the work on one or many projects
I want to be able to manage the issues in those projects. 

    

#### 4.3.1. Update existing Items
As an team member
In order to be able to manage new Items 
I want  to be able to create Items via the qto

    

#### 4.3.2. Remove existing Items
As an team member
In order to be able to stop the work on existing Items
I want to be able to remove Items via the qto

    

#### 4.3.3. Search for existing Items
As an team member
In order to be able to change attributes of the Items I am responsible for
I want to be able to update the Items' data. 

    

#### 4.3.4. Track Items progress
As an team member
In order to be able to quickly access existing Items
I want to be able to search the Items. 

    

#### 4.3.5. Track Items history
As a team member
In order to keep track on what and when was planned on daily basis
I want to be able to keep track what was planned on a project term  - day,week,month,year,quinquennial or decade  


    

### 4.4. Track issues relations
As a team member of a project
In order to trace the issues relations to userstories, features and tests or any other objects
I want to be able to access the related objects to an issue by means of a link

    

### 4.5. Items management
As a team member of the qto
In order to achieve the best posible efficiency during the work on one or many projects
I want to be able to manage the Items in those projects, where items could be ( problems , questions etc.

    

### 4.6. Measure success
As a team member
In order to measure the success of the planned issues 
I want  to be able to measure the deliverables of each issue by comparable metrics.

    

### 4.7. Monitor success
As a team member 
In order to monitor the success of the planned issues 
I want to  be able to monitor the metrics of the issues. 

    

### 4.8. Track issues relations
As a team member of a project
In order to trace the issues relations to userstories, features and tests or any other objects
I want to be able to access the related objects to an issue by means of a link

    

### 4.9. Time management
As an issues-manager 
In order to be prepared for issues such as ( events , tasks ) which have start and stop time
I want to  be able to save their start time and stop time per issue in every possible interface

    

#### 4.9.1. time centric planning
As an issues-manager 
In order to be able to plan the issues data for a certain term - day,week,month,year,quinquennial or decade
I want to be able to perform all the features of the qto on that specific period regardless whether it is today , in the past or in the future

    

#### 4.9.2. time centric reporting
As an issues-manager 
In order to be able to report the issues data for  a certain term - day,week,month,year,quinquennial or decade
I want to be able to perform all the features of the qto on that specific day regardless whether it is today , in the past or in the future

    

### 4.10. Project's persons issue combinations
As the project manager  of an qto project 
In order to be able to quickly and reliably combine the reported hours by the project's people
I want to be able to read their qto formatted google sheets and combine them into a single project's google qto sheet

    

## 5. PROJECT OBSERVER BIZ PERSPECTIVE
As a project observer 
In order to observe the advancement of a project  
I want to have a nice user experience while using the qto application.

    

### 5.1. Projects observation
As a project observer 
In order to observe the advancement of a project  
I want to be able to observe the project's data. 

    

### 5.2. Measure success
As a team member
In order to measure the success of the planned issues 
I want  to be able to measure the deliverables of each issue by comparable metrics.

    

### 5.3. Issues observation
As a project observer 
In order to observe the advancement of the project's issues  
I want to be able to observe the project's issues. 

    

### 5.4. Monitor success
As a team member 
In order to monitor the success of the planned issues 
I want to  be able to monitor the metrics of the issues. 

    

## 6. PRODUCT INSTANCE OWNER PERSPECTIVE
As a sysadmin of the qto application 
In order to complete the tasks and activities of my role
I want to have a nice user experience while using the qto application.

    

### 6.1. System deployability
As the SysAdmin 
In order to be able to provide access to a new database driven application to my organization
I want to be able to deploy an instance of the qto application and spawn a new project out of it in less than a hour from a clean Linux host including AWS.

    

### 6.2. Time management
As an issues-manager 
In order to be prepared for issues such as ( events , tasks ) which have start and stop time
I want to  be able to save their start time and stop time per issue in every possible interface

    

### 6.3. System stability
As the SysAdmin
In order to minimize downtimes and ensure continuous operations 
I want to the System containing the qto application to perform its defined functions on request without interruptions or unknown side effects

    

### 6.4. Generic CRUDS for items
As a team member
In order to be able to manage all the items in the application I have access to
I want to be able to create,update,delete and search for those items from the UI of the application.

    

### 6.5. System reliability
As the SysAdmin
In order to be able to rely on the operations of the tool
I want to the System containing the qto application to perform its functions as specified consistently 

    

### 6.6. Project's persons issue combinations
As the project manager  of an qto project 
In order to be able to quickly and reliably combine the reported hours by the project's people
I want to be able to read their qto formatted google sheets and combine them into a single project's google qto sheet

    

### 6.7. Ease of use
As the SysAdmin
In order to be efficient and decrease the amount of errors
I want to generally perform any command the system within the sysadmin scope via clean and memorable oneliners 

    

### 6.8. Security
As the PIO
In order to be able to provide the best possible security level of operation for the qto instance I am responsible for
I want to have pre-defined and clear set of tasks and activities to perform related to the security. 

    

#### 6.8.1. Password strength security
As the SysAdmin
In order to be assured that the hacking of the system via brut force attack will not be easy
I want the System to enforce password strength security by not allowing saving of weak passwords.

    

#### 6.8.2. Password reset enforceability
As the SysAdmin
In order to be assured that the hacking of the system via brut force attack will not be easy
I want the System to enforce password strength security by not allowing saving of weak passwords.

    

#### 6.8.3. Login info
As the PIO
In order to provide the best possible security for the data of my organisation 
I want to be able to provide easily and quickly answer to the question :
Who logged in and out when?

    

## 7. PROJECT OBSERVER BIZ PERSPECTIVE
As a project observer 
In order to observe the advancement of a project  
I want to have a nice user experience while using the qto application.

    

### 7.1. Projects observation
As a project observer 
In order to observe the advancement of a project  
I want to be able to observe the project's data. 

    

#### 7.1.1. Who logged in and out when
As the SysAdmin
In order to provide the best possible security for the data of my organisation 
I want to be able to provide easily and quickly answer to the question :
Who logged in and out when?

    

### 7.2. Issues observation
As a project observer 
In order to observe the advancement of the project's issues  
I want to be able to observe the project's issues. 

    

#### 7.2.1. Who changed what and why
As the SysAdmin
In order to provide the best possible security for the data of my organisation 
I want to be able to provide easily and quickly answer to the question :
Who changed what and why?

    

## 8. ETL AND INTEGRATIONS PERSPECTIVE
As an ETL and integrations specialist 
In order to complete the tasks and activities of my role
I want to have a nice user experience while using the qto application.

    

### 8.1. Database to json files data load
As the ETL and Integration Specialist 
In order to be able to quickly move all the project data into a different storage format 
I want to be able to export the project db data into json files - one per table via a single shell call. 

    

### 8.2. Database metadata export
As the ETL and Integration Specialists
In order to be able to show, communicate and store the meta-data of my application
I want to be able to export the project's database meta-data into an xls file.

    

### 8.3. System performance
As the SysAdmin 
In order to ensure the performance of the qto application
I want to the System containing the qto application to perform its functions within the defined performance criteria.

    

### 8.4. Json files to db data load
As the ETL and Integration Specialist 
In order to be able to quickly move all the project data from json files into the db tables
I want to be able to import the exported json files ( one per table ) into the database. 

    

### 8.5. Xls-to-postgres-db hierarchical data load
As the Data Integrator
In order to be efficient while handling the System's hierarchical data
 I want to be able to
use a single shell call to load all or chosen table(s) to the postgres db 

    

#### 8.5.1. error reporting in xls-to-postgres-db hierarchical data load 
As the Data Integrator
In order to be efficient while troubleshooting data loading errors 
I want to be able to see : 
 - which table's load failed
 - what was the error in failed to 

    

### 8.6. Project deployment
As a potential Developer of the qto or a qto application 
In order to have the easiest installation experience 
I want to be able to deploy fully functional development environment on a clean host in less than an hour with a single oneliner. 

    

## 9. DEVOPS ENGINEER PERSPECTIVE
As a qto devops engineer
In order to be efficient and decrease the amount of errors
I want to generally perform any command the system within the sysadmin scope via clean and memorable oneliners 

    

### 9.1. Documentation


    

### 9.2. ETL


    

#### 9.2.1. Quick start documentation
As a Full-Stack Developer
In order to be able to quickly start on my own hacking the project
I want to have a quick start documentation, which will simply guide me to start on my own.

    

#### 9.2.2. Application's source code and documentation integrity
As a Full-Stack Developer
In order to make easy the entry of other developers to the projects
I want to be able to point to written documentation for user-stories, issues, features and functionalities, which will be linked to parts of the source code. 

    

### 9.3. Installation


    

#### 9.3.1. A single shell call installation
As a potential Developer of the qto or a qto application 
In order to have the easiest installation experience 
I want to be able to deploy fully functional development environment on a clean host in less than an hour with a single oneliner. 

    

#### 9.3.2. A reliable installation documentation
As a potential DevOps engineer of the qto or a qto application 
In order to have the easiest installation experience 
I want to have a reliable documentation, which will have short commands 

    

### 9.4. Testing
As a qto devops engineer 
In order to be able to rely on the operations of the tool 
 and manage easily its features and functionalities
I want to the easily verify and  test parts or the whole System 
by issuing a single shell call.

    

#### 9.4.1. Clarity and brevity of the end to end tests
As an ITOPS 
In order to be able to verify all the features and functionalities of the tool within the System
I want to see the results of each test in 1 flow in the following format:.

    

#### 9.4.2. Databases - local to remote and remote to local sync
As a DevOps Operator
In order to :
 - keep the dog-fooded local and remote db instances in sync
 - and have full mobile access via the web 
I want to :
 - be able to apply a full db dump remotely via a single one-liner 
 - both local to remote and remote to local



    

#### 9.4.3. Abort end-to-end tests on single test fail
As an ITOPS 
In order to be able to run continuously end-to-end tests and skip for several runs failing tests
I want to be able to configure the single e2e entry point script to skip certain tests, but report me what was skipped. 

    

#### 9.4.4. Single shell call for e2e testing
As an DevOps
In order to be able to verify all the features and functionalities of the tool within the System
I want to run a single shell call running all the end-to-end test of the application ensuring the specified features and functionalities.  

    

### 9.5. ETL
As an DevOps
In order to be able to quickly manage the combination of binaries, configuration, own source code and data 
I want to be able to quickly and efficiently Extract, Transform and Load data from and into any qto based or related System.

    

#### 9.5.1. Database to json files data load
As the ETL and Integration Specialist 
In order to be able to quickly move all the project data into a different storage format 
I want to be able to export the project db data into json files - one per table via a single shell call. 

    

#### 9.5.2. Items data transformations
As a cli user of the qto application 
In order to be able to sort the issues according to their attributes 
and edit them in both txt file and xls file
I want to be able to perform the following loads: 
txt-to-db - to load a txt file with issues to an issues table in db
db-to-xls - to load a xls file from db table to xls
xls-to-db - to load a xls file with issues to an issues table in db

    

#### 9.5.3. Json files to db data load
As the ETL data engineer 
In order to be able to quickly move all the project data from json files into the db tables
I want to be able to import the exported json files ( one per table ) into the database. 

    

#### 9.5.4. Xls-to-postgres-db hierarchical data load
As the Data Integrator
In order to be efficient while handling the System's hierarchical data 
I want to be able to
use a single shell call to load all or chosen table(s) to the mysql db 

    

#### 9.5.5. Log entries format configuration
As a Full-Stack Developer
In order to be able to get the msg of any component of the application 
I want each log entry to content:
- the type of the entry - log , error, warn , fatal
- the timestamp of the log entry event 
- the name of the component issueing the msg and the line num of the src file 
- the msg as it was echoed by the application

    

#### 9.5.6. Tool run log to human readable description
As a CLI user 
In order to be able to get a human readable description of the log of the specific run of the tool
I want to be able to translate the recorded uuid's in the execution run log to their respective records

    

#### 9.5.7. Components start run message print
As a CLI user 
In order to know when a component has been started
I want to be able to see the "START &lt;&lt;COMPONENT NAME&gt;&gt; on either the STDOUT or the log file of the component

    

#### 9.5.8. Tool exit with exit code and exit message
As a CLI user or calling  automated component
In order to be able to understand whether or not the execution of the call to the tool was successful or not 
I want to get the exit code from the tool execution and see the exit message

    

### 9.6. Run-time management


    

#### 9.6.1. Logging
As a Full-Stack Developer
In order to quickly understand what is happening in the application
I want to have easy-to-use and highly customisable logging to both file and console.

    

#### 9.6.2. Single shell call for projects switching
As a qto DevOps engineer
In order to be able to switch between different projects quickly in the shell 
I want to be able to issue a single shell call for loading a project's configuration
and run the qto tool against the predefined configuration of an qto project's configuration.

    

#### 9.6.3. Start and Stop of the Application
As a qto DevOps engineer
In order to be able to quickly start and stop the qto application
I want to have a single shell call for those actions, with no other configuration, run-time or binary dependencies.

    

### 9.7. Security


    

#### 9.7.1. Security modes
As a qto DevOps engineer
In order to be able to effortlessly develop the application
I want to have 3 Security Modes, which could be changed during the restart of the application, by a simple ENV var:
 - No Auth - no auth whatsoever - so that I could be able to concentrate on non-security issues
 - Simple Native Auth mode - every team member can do anything on each item ( excepts the users)
 - RBAC Native Auth mode - configurability via db for which role, can perform what action on which item 

    

## 10. UI DEVELOPER PERSPECTIVE
As the UI Developer
In order to be able to deliver working solutions for the UI 
I want to have user friendly development experience. 

    

### 10.1. Foreign Keys (FK) configurability
As the UI Developer 
In order to be able to be able to quickly display FK table values as drop boxes 
I want to be able to configure from the app_item_attributes table :
 - the table from which the drop box will fetch it's values ( if not defined use the &lt;&lt;table&gt;&gt;_guid
 - the attribute name from which the drop down list will be built

    

### 10.2. Code traceability by uuid
As the UI Developer 
In order to be able to grasp the inner working of the application
I want to be able to search by user-story uuid from the source code of the application

    

### 10.3. Single shell call for projects switching
As an issues-manager 
In order to be able to switch between different projects quickly
I want to be able to issue a single shell call for loading a project's configuration
and run the issue-handler against this preloaded configuration

    

### 10.4. Issues publishing from shell calls
As a DevOps 
In order to be able to quickly share the current issues data in tabular format
I want to be able to issue a single shell call for copying the current items data to a medium by specifying the tables to be published

    

### 10.5. Metadata handling
As a DevOps operator
In order to be able to programatically manage all aspects of my data
I wan to to have a single entry point to manage the meta data per tables , columns and UI elements
so that even a table, column or whatever object is not populated in the meta still there will be default values for it usable by the application.

    

### 10.6. Easy setup for testability
As the UI Developer
In order to deliver working ui units 
I want  to be able to quickly setup the existing project with minimalistic default set of data

    

<file_sep>-- DROP TABLE IF EXISTS monthly_issues_202005 ; 

SELECT 'create the "monthly_issues_202005" table'
; 

CREATE TABLE public.monthly_issues_202005 (
    guid uuid DEFAULT public.gen_random_uuid() NOT NULL,
    id bigint DEFAULT (to_char(CURRENT_TIMESTAMP, 'YYMMDDHH12MISS'::text))::bigint NOT NULL,
    category_guid uuid DEFAULT '70724562-d83c-446e-94cf-58ced84f3a0e'::uuid NOT NULL,
    issues_status_guid uuid DEFAULT 'cb989a14-d0b8-46e4-b2cc-5e2a974b5d29'::uuid NOT NULL,
    prio integer DEFAULT 0 NOT NULL,
    name character varying(100) DEFAULT 'name...'::character varying NOT NULL,
    description character varying(4000),
    app_users_guid uuid DEFAULT public.gen_random_uuid(),
    type character varying(30) DEFAULT 'task'::character varying NOT NULL,
    update_time timestamp without time zone DEFAULT date_trunc('second'::text, now())
);

create unique index idx_uniq_monthly_issues_202005_id on monthly_issues_202005 (id);

-- the rank search index
CREATE INDEX idx_rank_monthly_issues_202005 ON monthly_issues_202005
USING gin(to_tsvector('English', name || ' ' || description || 'owner')); 


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.monthly_issues_202005'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 

--The trigger:
CREATE TRIGGER trg_set_update_time_on_monthly_issues_202005 BEFORE UPDATE ON monthly_issues_202005 
   FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'monthly_issues_202005'::regclass;

<file_sep># src/bash/qto/funcs/clone-to-app.test.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doTestCloneToApp comments ...
# ---------------------------------------------------------
doTestCloneToApp(){

	do_log "DEBUG START doTestCloneToApp"
	
	# add your action implementation code here ... 
   bash src/bash/qto/qto.sh -a to-app=qto

	do_log "DEBUG STOP  doTestCloneToApp"
}
# eof func doTestCloneToApp


# eof file: src/bash/qto/funcs/clone-to-app.test.sh
<file_sep># src/bash/qto/funcs/print-help.test.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doTestPrintHelp comments ...
# ---------------------------------------------------------
doTestPrintHelp(){

	do_log "DEBUG START doTestPrintHelp"
	
	cat doc/txt/qto/tests/print-help.test.txt
	sleep 2
	# add your action implementation code here ... 

	do_log "DEBUG STOP  doTestPrintHelp"
}
# eof func doTestPrintHelp


# eof file: src/bash/qto/funcs/print-help.test.sh
<file_sep># src/bash/qto/funcs/xls-to-db.test.sh

# v1.0.9
# ---------------------------------------------------------
# cat doc/txt/qto/tests/xls-to-db.test.txt
# ---------------------------------------------------------
doTestXlsToDb(){

	do_log "DEBUG START doTestXlsToDb"
	
	
	sleep "$sleep_interval"
	# Action !!!
   export load_model='upsert'
   export rdbms_type='postgres'
   src/bash/qto/qto.sh -a xls-to-db -t monthly_issues,daily_issues
   exit_code=$?
   sleep "$sleep_interval"
   test $exit_code -ne 0 && return
   
	# Action !!!
   export load_model='nested-set'
   src/bash/qto/qto.sh -a xls-to-db -t devops_guide_doc,userstories_doc
   exit_code=$?
   sleep "$sleep_interval"
   test $exit_code -ne 0 && return

	do_log "DEBUG STOP  doTestXlsToDb"
}


# eof file: src/bash/qto/funcs/xls-to-db.test.sh
<file_sep># 
do_chk_install_redis(){
   
   which redis-cli || {
      sudo apt-get update
      sudo adduser --system --group --no-create-home redis
      sudo apt-get install -y redis-server

      sudo mkdir -p /var/lib/redis/
      sudo chown -R redis:redis /var/lib/redis
      sudo chmod 770 /var/lib/redis
      sudo cp -v /etc/redis/redis.conf /etc/redis/redis.conf.orig

      # capture the first ip address into a var to add the the bind directive
      export my_ip=$(hostname -I|awk '{print $1}')

      # append the following lines to the /etc/redis/redis.conf file
      echo "bind $my_ip"| sudo tee -a /etc/redis/redis.conf
      echo 'supervised systemd' | sudo tee -a /etc/redis/redis.conf

      # comment out the bind directive, having it bound on 127.0.0.1 breaks Redis
      sudo perl -pi -e 's/bind 127.0.0.1 ::1/# bind 127.0.0.1 ::1/ig' /etc/redis/redis.conf
      while read -r file ; do
         IFS='' read -r -d '' perl_code <<"EOF_PERL_CODE"
            use strict; use warnings; binmode STDOUT, ":utf8"; use utf8; use JSON; use Data::Printer;
            my $my_ip = $ENV{'my_ip'};
            my $sjson;
            {
              local $/; open my $fh, "<", $ARGV[0];
              $sjson = <$fh>; close $fh;
            }
            my $data = decode_json($sjson);
            #p $data ; 
            $data->{'env'}->{'redis'}->{'server'} = $my_ip; 
            my $json = JSON->new->allow_nonref;
            open my $fh, ">", $ARGV[0];
            print $fh $json->pretty->encode($data);
            close $fh;
EOF_PERL_CODE
      
         perl -e "$perl_code" $file
	   done < <(find $PRODUCT_DIR/cnf/env/ -type f| grep -v '.bak')

      # restart to apply the changes 
      sudo systemctl restart redis
      sudo systemctl restart redis.service

      # check that redis is running
      sudo ps -ef | grep -v grep | grep -i redis

      # should return PONG
      redis-cli ping
   }
}
<file_sep># how to add files to the git 
clear ; git log --pretty --format='%h %ae %<(15)%an ::: %s'
alias git='GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa.ysg" git'
git add --all ; git commit -m "$git_msg" --author "<NAME> <<EMAIL>" # --amend
GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa.ysg" git push
GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa.ysg" git pull

# OBS AMMEND AND git push FORCE !!
git add --all ; git commit -am "$git_msg" --author "<NAME> <<EMAIL>>" \
   --amend && git push --force
<file_sep># src/bash/qto/funcs/run-container.func.sh

# v0.6.7
doRunContainer(){
   do_log "DEBUG START doRunContainer"

   test -z ${qto_project:-} && \
      source "$PRODUCT_DIR/lib/bash/funcs/parse-cnf-env-vars.sh" && \
      doParseCnfEnvVars $PRODUCT_DIR/cnf/$RUN_UNIT.$ENV.$host_name.cnf

   tgt_PRODUCT_DIR='/opt/csitea/'"$RUN_UNIT/$environment_name"
   chmod 777 $PRODUCT_DIR/src/bash/qto/install/docker/docker-entry-point.sh

   container_name='qto-container-'$(date "+%Y%m%d_%H%M%S")

   docker run -d --name $container_name \
      -e PRODUCT_DIR=$tgt_PRODUCT_DIR \
      -e host_host_name=$(hostname -s) \
      -v $PRODUCT_DIR:$tgt_PRODUCT_DIR \
      -p  127.0.0.1:${MOJO_MORBO_PORT:-}:$MOJO_MORBO_PORT \
      'qto-image':${product_version:-}.$ENV
   
   # -p 127.0.0.1:${postgres_rdbms_port:-}:$postgres_rdbms_port \

   printf "\n"
   echo do check which containers are running 
   echo docker container ls
   printf "\n\n"
   echo "to attach to the running container run:"
   echo "docker exec -it  $container_name /bin/bash"
   printf "\n\n"
   echo do stop it run the following command
   echo docker container stop $container_name

   do_log "DEBUG STOP  doRunContainer"
}


# eof file: src/bash/qto/funcs/run-container.func.sh
<file_sep>
do_chk_setup_bash(){
   bash_opts_file=~/.bash_opts.$host_name
   cp -v $PRODUCT_DIR/cnf/bash/.profile_opts.host-name ~/.profile_opts.$host_name
   cp -v $PRODUCT_DIR/cnf/bash/.bash_opts.host-name $bash_opts_file

   set +e
   test $(grep -c "source $bash_opts_file" ~/.bashrc) -lt 1 && \
      cat <<EOF | sudo tee -a ~/.bashrc > /dev/null 
   source $bash_opts_file
EOF

}
<file_sep># src/bash/qto/funcs/generate-md-docs.func.sh

#
# ---------------------------------------------------------
# for docs: cat doc/txt/qto/funcs/generate-md-docs.func.txt
# ---------------------------------------------------------
doGenerateMdDocs(){
   do_log "DEBUG START doGenerateMdDocs"
   test -z "${PROJ_INSTANCE_DIR-}" && PROJ_INSTANCE_DIR="$PRODUCT_DIR"
   test -z "${docs_root_dir-}" && docs_root_dir="$PROJ_INSTANCE_DIR"

   test -z ${PROJ_CONF_FILE-} && PROJ_CONF_FILE=$PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json
   do_export_json_section_vars $PROJ_CONF_FILE '.env.app'

   # <<web-host>>:<<web-port>>/<<db>>/select/export_files?as=grid&od=id
   basic_url="$ht_protocol://${web_host:-}:${port:-}/${postgres_app_db:-}"
   furl="$basic_url"'/select/export_files?as=grid&od=id&pg-size=20'
   echo "running: curl --cookie ~/.qto/cookies.txt --insecure  -s $furl \| jq -r '.dat[]|.url'"
   curl --cookie ~/.qto/cookies.txt --insecure  -s $furl | jq -r '.dat[]|.url'
   ret=$?
   test $ret != "0" && do_exit $ret "failed to get data from the $furl"

   while read -r url ; do 
      file_name=$(curl --cookie ~/.qto/cookies.txt --insecure  -s $furl | jq -r '.dat[]| select(.url=='\"$url\"')| .name'); 
      file_name="$file_name.md"
      rel_path=$(curl --cookie ~/.qto/cookies.txt --insecure  -s $furl | jq -r '.dat[]| select(.url=='\"$url\"')| .path');
      rel_path="$rel_path/md"
      [ "${rel_path-}" == "null/md" ] && rel_path=""
      mkdir -p $docs_root_dir/$rel_path
      file_path=$(echo $docs_root_dir/$rel_path/$file_name|perl -ne 's|[/]{2,5}|/|g;print')
      url="$url"'?as=md&type=msft'
      echo -e "\nrunning: curl --cookie ~/.qto/cookies.txt --insecure -s -o \"$file_path\" \ \n \"$basic_url/export/$url\""       
      curl --cookie ~/.qto/cookies.txt --insecure  -s "$basic_url/export/$url" -o "$file_path"
      lines=$(wc -l "$file_path"|awk '{print $1;}')
      [[ $lines -eq 0 ]] && do_exit 0 "load the $url !!! the table is empty !!!"
      echo -e "$lines lines in the $file_path file \n"
   done < <(curl --cookie ~/.qto/cookies.txt --insecure  -s $furl | jq -r '.dat[]|.url')
	
   do_flush_screen
   do_log "DEBUG STOP  doGenerateMdDocs"
}

# eof file: src/bash/qto/funcs/generate-md-docs.func.sh
<file_sep># vim:set ft=dockerfile:
FROM ubuntu:18.04

ARG PRODUCT_DIR
ENV PRODUCT_DIR $PRODUCT_DIR
ARG postgres_app_db 
ENV postgres_app_db $postgres_app_db
ARG postgres_sys_usr_admin='usrqtoadmin'
ENV postgres_sys_usr_admin_pw '<PASSWORD>'
ARG host_host_name
ENV host_host_name $host_host_name

# obs !!! todo: parametrize qto-190616104728
RUN echo "root:root" | chpasswd
RUN echo 'export PS1="`date "+%F %T"` \u@\h  \w \\n\\n  "' >> /root/.bashrc

# install the most basic binaries
ARG TZ
ENV TZ $TZ
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone 
RUN apt-get update && apt-get install -y wget curl sudo perl zip unzip gnupg2 gnupg1 git bash jq vim

WORKDIR /opt/
ADD . $PRODUCT_DIR
WORKDIR $PRODUCT_DIR
#   ^^^ todo:ysg replace with the git clone 

RUN chmod 755 src/bash/qto/install/docker/install-admin-utils.sh && \
    bash src/bash/qto/install/docker/install-admin-utils.sh
RUN chmod 755 src/bash/qto/install/docker/provision-os-users.sh && \
    bash src/bash/qto/install/docker/provision-os-users.sh
RUN chmod 755 src/bash/qto/install/docker/install-postgres.sh && \
    bash src/bash/qto/install/docker/install-postgres.sh

# add VOLUMEs to allow backup of config, logs and databases
# VOLUME  ["/etc/postgresql", "/var/log/postgresql", "/var/lib/postgresql"]

# set the default command to run when starting the container
RUN chmod 755 src/bash/qto/install/docker/install-perl-modules.sh && \
   bash src/bash/qto/install/docker/install-perl-modules.sh

RUN chown -R usrqtoadmin:grpqtoadmin "/opt/csitea" && \
    chmod 777 "/opt" && \
    chmod 755 "$PRODUCT_DIR/src/bash/qto/bootstrap-qto-docker.sh"


USER usrqtoadmin
WORKDIR $PRODUCT_DIR
RUN $PRODUCT_DIR/src/bash/qto/bootstrap-qto-docker.sh

USER root
RUN rm -vr /opt/csitea && chmod 755 "/opt"

USER usrqtoadmin
WORKDIR $PRODUCT_DIR

# CMD ["bash","-c","while true; do sleep 1; done;"]
CMD bash -c $PRODUCT_DIR/src/bash/qto/install/docker/docker-entry-point.sh
<file_sep>-- DROP TABLE IF EXISTS logs ; 

SELECT 'create the "logs" table'
; 
   CREATE TABLE logs (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             numeric (25) UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISSMSUS') as numeric(25)) 
    , prio           integer NOT NULL DEFAULT 5
    , env            varchar (50) NOT NULL DEFAULT 'dev'
    , app            varchar (50) NOT NULL DEFAULT 'qto'
    , name           varchar (200) NOT NULL DEFAULT 'name ...'
    , description    varchar (400)
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_logs_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.logs'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 

--The trigger:
CREATE TRIGGER trg_set_update_time_on_logs BEFORE UPDATE ON logs FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'logs'::regclass;

<file_sep>---- DROP TABLE IF EXISTS links ; 

SELECT 'create the "links" table'
; 
   CREATE TABLE links (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , name           varchar (100) NOT NULL DEFAULT 'a-name...'
    , url            varchar (200) NOT NULL DEFAULT 'a-url...'
    , description    varchar (200) NOT NULL DEFAULT 'a-description...'
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_links_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

create unique index idx_links_uniq_id on links (id);


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.links'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 

--The trigger:
CREATE TRIGGER trg_set_update_time_on_links BEFORE UPDATE ON links FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'links'::regclass;
<file_sep>doChangeVersion(){

	tgt_version="$1" ; shift 1;
	prefix='to-ver='
	tgt_version=${tgt_version#$prefix}
	tgt_env_name=$(echo $environment_name | perl -ne "s/$product_version/$tgt_version/g;print")
	tgt_env_name=$(echo $tgt_env_name | perl -ne "s/$ENV/dev/g;print") # ALWAYS !!!
	tgt_PRODUCT_DIR=$product_dir/$tgt_env_name
	mkdir -p $tgt_PRODUCT_DIR	

	test "$tgt_PRODUCT_DIR" == "$PRODUCT_DIR" && return
	rm -fvr $tgt_PRODUCT_DIR/*
	test $? -eq 0  || do_exit 2 "cannot write to $tgt_PRODUCT_DIR !"
	
	doCreateRelativePackage
	unzip -o $zip_file -d $tgt_PRODUCT_DIR ; cp -v $zip_file $tgt_PRODUCT_DIR

   # change the current version str to tgt-version in the configuration dir 
   export to_srch=$product_version
   export to_repl=$tgt_version
   export dir_to_morph=$tgt_PRODUCT_DIR/cnf
   doMorphDir # :fin morph-dir.func.sh

	# ensure that all the files in the target product version dir are indentical to the current ones
	while read -r file ; do (
		do_log "DEBUG comparing src file: $file"
		do_log "DEBUG to tgt file: $tgt_PRODUCT_DIR/$file"
		test -f "$PRODUCT_DIR/$file" \
			&& diff "$PRODUCT_DIR/$file" "$tgt_PRODUCT_DIR/$file"
		[[ $? != 0 ]] && exit $?
	);
	done < <(cat $include_file)

   cd $tgt_PRODUCT_DIR # to create default links
   ln -sf `pwd`/src/bash/qto/qto.sh qto ; sudo chmod 754 `pwd`/src/bash/qto/qto.sh
   ln -sf `pwd`/src/perl/qto/script/qto.pl qto.pl ; sudo chmod 754 `pwd`/src/perl/qto/script/qto.pl
   cd -
}
<file_sep>do_chk_provision_nginx(){
   set +e
   # chk: devops_guide_doc-200325051051
   #sudo mkdir /etc/systemd/system/nginx.service.d
   cat << EOF | sudo tee /etc/systemd/system/nginx.service.d/override.conf
[Service]
ExecStartPost=/bin/sleep 0.1
EOF

   sudo mkdir -p /etc/nginx/sites-available
   sudo mkdir -p /etc/nginx/sites-enabled
   sudo test -f  /etc/nginx/sites-available/* && {
      sudo rm -v /etc/nginx/sites-available/*
   }
   sudo test -f  /etc/nginx/sites-enabled/* && {
      sudo rm -v /etc/nginx/sites-enabled/*
   }

   # qto-200326084258 create the cache dir NOT in the global dir, related to site crashes
   mkdir -p ~/var/cache ; chmod -R 777 ~/var/cache
   perl -pi -e 's|~/|'$HOME/'|g' $PRODUCT_DIR/cnf/nginx/etc/nginx/nginx.conf #qto-200402172803
   sudo cp -v $PRODUCT_DIR/cnf/nginx/etc/nginx/nginx.conf /etc/nginx/nginx.conf

   source $PRODUCT_DIR/lib/bash/funcs/export-json-section-vars.sh
   for env in `echo dev tst prd`; do \

      do_export_json_section_vars $PRODUCT_DIR/cnf/env/$env.env.json '.env.app'
      sudo cp -v $PRODUCT_DIR/cnf/nginx/etc/nginx/sites-available/%env%.http-site.conf \
         /etc/nginx/sites-available/$env.http-site.conf
      sudo perl -pi -e 's|\%nginx_port\%|'"$nginx_port"'|g' "/etc/nginx/sites-available/$env.http-site.conf"
      sudo perl -pi -e 's|\%https_port\%|'"$https_port"'|g' "/etc/nginx/sites-available/$env.http-site.conf"
      sudo perl -pi -e 's|\%mojo_hypnotoad_port\%|'"$mojo_hypnotoad_port"'|g' "/etc/nginx/sites-available/$env.http-site.conf"
      sudo perl -pi -e 's|\%web_host\%|'"$web_host"'|g' "/etc/nginx/sites-available/$env.http-site.conf"
      sudo perl -pi -e 's|\%port\%|'"$port"'|g' "/etc/nginx/sites-available/$env.http-site.conf"
      
      sudo ln -fs /etc/nginx/sites-available/$env.http-site.conf /etc/nginx/sites-enabled/$env.http-site.conf
      sudo ls -la /etc/nginx/sites-enabled/$env.http-site.conf
   done ;

   sudo chown -R www-data:www-data /etc/nginx # !!! monthly_issues_202003-200326084258
   sudo systemctl daemon-reload
   sudo systemctl try-restart nginx
   sudo systemctl enable nginx
   sudo systemctl status nginx
   source $PRODUCT_DIR/.env
   do_export_json_section_vars $PRODUCT_DIR/cnf/env/$ENV.env.json '.env.app'
  

}
<file_sep># src/bash/qto/funcs/print-usage.help.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doHelpPrintUsage comments ...
# ---------------------------------------------------------
doHelpPrintUsage(){

	do_log "DEBUG START doHelpPrintUsage"
	
	cat doc/txt/qto/helps/print-usage.help.txt
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!

	do_log "DEBUG STOP  doHelpPrintUsage"
}
# eof func doHelpPrintUsage


# eof file: src/bash/qto/funcs/print-usage.help.sh
<file_sep># src/bash/qto/funcs/mojo-morbo-stop.test.sh

# v0.6.7
# ---------------------------------------------------------
# test the stoping of the mojo morbo server
# ---------------------------------------------------------
doTestMojoMorboStop(){

	sleep "$sleep_interval"
	
   # Action !!!
   bash src/bash/qto/qto.sh -a mojo-morbo-stop
	do_log "check with netstat "
   # sudo visudoers 
   # usrqtoadmin ALL=(ALL) NOPASSWD: /bin/netstat -tulpn
   sudo netstat -tulpn | grep qto

}


# eof file: src/bash/qto/funcs/mojo-morbo-stop.test.sh
<file_sep>#!/usr/bin/env bash
# purpose: a SLIGHTLY opinionated bash,tmux,vim and git setup 
# usage:
# curl https://raw.githubusercontent.com/YordanGeorgiev/qto/master/src/bash/qto/scripts/pre-push/setup-bash-n-vim.sh | bash -s <EMAIL> ysg

main(){
   do_enable_locate
   do_set_vars
   do_chk_provision_tmux
   do_chk_provision_vim
   do_chk_provision_git
   do_chk_provision_ssh_keys
	do_fake_history
   do_chk_provision_bash
	do_echo_copy_pasteables
}


do_enable_locate(){
   sudo updatedb & # because of the locate elflord.vim below and just to speed up..
}


do_set_vars(){
	email=${1:-<EMAIL>}
	my_initials=${1:-ysg}
   host_name=`hostname -s`
}


do_chk_provision_tmux(){

   echo 'start ::: provisioning tmux'
   which tmux 2>/dev/null || {
      sudo apt-get update
      sudo apt-get install -y tmux
   }
   mkdir -p ~/.tmux/plugins
   git clone https://github.com/tmux-plugins/tpm ~/.tmux/plugins/tpm
   git clone https://github.com/tmux-plugins/tpm ~/.tmux/plugins/tmux-copycat
   echo verify the tmux plugins
   find ~/.tmux/plugins -type d -maxdepth 2

   test -f ~/.tmux.conf && cp -v ~/.tmux.conf ~/.tmux.conf.$(date "+%Y%m%d_%H%M%S")
   wget -O ~/.tmux.conf \
      https://raw.githubusercontent.com/YordanGeorgiev/ysg-confs/master/.tmux.conf.host-name
   echo 'stop  ::: provisioning tmux'

}


do_chk_provision_vim(){

   echo 'start ::: provisioning vim'
   which vim 2>/dev/null || {
      sudo apt-get update
      sudo apt-get install -y vim
   }
   which wget 2>/dev/null || {
      sudo apt-get update
      sudo apt-get install -y wget
   }
   test -f ~/.vimrc && cp -v ~/.vimrc ~/.vimrc.$(date "+%Y%m%d_%H%M%S")
   wget -O ~/.vimrc 'https://raw.githubusercontent.com/YordanGeorgiev/ysg-confs/master/.vimrc.host-name'

   # set the ngix syntax highlighting 
   mkdir -p ~/.vim/syntax/
   cd ~/.vim/syntax/
   wget http://www.vim.org/scripts/download_script.php?src_id=19394
   mv download_script.php\?src_id\=19394 nginx.vim
   cat > ~/.vim/filetype.vim <<EOF_NGINX
   au BufRead,BufNewFile /etc/nginx/*,/usr/local/nginx/conf/* if &ft == '' | setfiletype nginx | endif
EOF_NGINX

   # add the perl.vim
   mkdir -p ~/.vim/ftplugin
   wget -O ~/.vim/ftplugin/perl.vim \
      'https://raw.githubusercontent.com/YordanGeorgiev/ysg-confs/master/.vim/ftplugin/perl.vim'

   # add the smartcom 
   mkdir -p ~/.vim/plugin
   wget -O ~/.vim/plugin/smartcom.vim \
      'https://raw.githubusercontent.com/YordanGeorgiev/ysg-confs/master/.vim/plugin/smartcom.vim'

   sudo cp -rv ~/.vim/ /root/
   cd -

   # set the grey color on the nums on the left ...
   sudo perl -pi -e \
      '$_="$_\nhi LineNr term=bold cterm=NONE ctermfg=DarkGrey ctermbg=NONE gui=NONE guifg=DarkGrey guibg=NONE"
      if $. == 27' $(locate elflord.vim) &
   echo 'stop ::: provisioning vim'

}


do_chk_provision_git(){

   echo 'start ::: provisioning git'
   test -f ~/.gitconfig && cp -v ~/.gitconfig ~/.gitconfig.$(date "+%Y%m%d_%H%M%S")

cat << EOF_GIT >> ~/.gitconfig

   [credential]
     helper = cache

   [core]
     editor = vim
     pager = less -r
     autocrlf = false

   [user]
      name = <NAME>
      email = $email

   [push]
      default = simple
      followTags = true

   [color]
     diff = auto
     status = auto
     branch = auto
     interactive = auto
     ui = true
     pager = true

   [color "status"]
     added = green
     changed = red bold
     untracked = magenta bold

   [color "branch"]
     remote = yellow

   [fetch]
      prune = true
EOF_GIT

   echo 'stop ::: provisioning git'

}


do_chk_provision_ssh_keys(){

   which expect || sudo apt-get update && sudo apt-get install -y expect

   # if the ssh key does not exist create it ...
   test -f ~/.ssh/id_rsa.ysg.pub.`hostname -s` || {
   expect <<- EOF_EXPECT
      set timeout -1
      spawn ssh-keygen -t rsa -b 4096 -C $email -f $HOME/.ssh/id_rsa.ysg.$host_name
      expect "Enter passphrase (empty for no passphrase): "
      send -- "\r"
		expect "Enter same passphrase again: "
      send -- "\r"
      expect eof
EOF_EXPECT

   }

}

do_chk_provision_bash(){

   echo 'start ::: fetching bash_opts'
   wget -O ~/.bash_opts.`hostname -s` \
      'https://raw.githubusercontent.com/YordanGeorgiev/ysg-confs/master/.bash_opts.host-name'

   echo 'stop  ::: fetching bash_opts'

}


do_fake_history(){

	cat << 'EOF_HIS' >> ~/.bash_history
ssh-keygen -t rsa -b 4096 -C "<EMAIL>" -f ~/.ssh/id_rsa.ysg.`hostname -s`
clear ; git log --pretty --format='%h %<(15)%ae %<(15)%an ::: %s'
alias git='GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa.ysg.`hostname -s`" git'
git add --all ; git commit -m "$git_msg" --author "<NAME> <<EMAIL>"; git push
git reset --hard origin/$(git rev-parse --abbrev-ref HEAD)
curr_branch=$(git rev-parse --abbrev-ref HEAD); git branch "$curr_branch"--$(date "+%Y%m%d_%H%M"); git branch -a | grep $curr_branch | sort -nr | less
EOF_HIS

}


do_echo_copy_pasteables(){
	cat ~/.ssh/id_rsa.ysg.$host_name.pub
	echo EOF ~/.ssh/id_rsa.ysg.$host_name.pub
	echo -e '\n\n'
   echo "source ~/.bash_opts.$host_name"
	echo -e '\n\n'
}


main # and .... Action !!!

# here is your parameter expansions cheat ;o) 
# +--------------------+----------------------+-----------------+-----------------+
# |                    |       parameter      |     parameter   |    parameter    |
# |                    |   Set and Not Null   |   Set But Null  |      Unset      |
# +--------------------+----------------------+-----------------+-----------------+
# | ${parameter:-word} | substitute parameter | substitute word | substitute word |
# | ${parameter-word}  | substitute parameter | substitute null | substitute word |
# | ${parameter:=word} | substitute parameter | assign word     | assign word     |
# | ${parameter=word}  | substitute parameter | substitute null | assign word     |
# | ${parameter:?word} | substitute parameter | error, exit     | error, exit     |
# | ${parameter?word}  | substitute parameter | substitute null | error, exit     |
# | ${parameter:+word} | substitute word      | substitute null | substitute null |
# | ${parameter+word}  | substitute word      | substitute word | substitute null |
# +--------------------+----------------------+-----------------+-----------------+
<file_sep>define stop-img-containers
	@clear
	-@docker container stop $$(docker ps -aqf "name=${product}-${1}-con") 2> /dev/null
	-@docker container rm $$(docker ps -aqf "name=${product}-${1}-con") 2> /dev/null
endef<file_sep># src/bash/qto/funcs/run-benchmarks.func.sh

# v0.6.8
# ---------------------------------------------------------
# implement the calls to all the functional tests
# ---------------------------------------------------------
doRunBenchmarks(){
  
   export QTO_NO_AUTH=1
   test -z ${qto_project:-} && \
      source "$PRODUCT_DIR/lib/bash/funcs/parse-cnf-env-vars.sh" && \
      doParseCnfEnvVars "$PRODUCT_DIR/cnf/$RUN_UNIT.$ENV.*.cnf"
   
   do_log "INFO START testing controllers"
   while read -r f ; do 
      do_log "INFO START functional test for $f"
      perl $f ;
      do_log "INFO STOP  functional test for $f"
      do_flush_screen
   done < <(find src/perl/qto/t/lib/Qto/App/benchmarks -type f -name '*.pl'|sort)

   export QTO_NO_AUTH=0
}


<file_sep># src/bash/qto/funcs/gmail-package.test.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doTestGmailPackage comments ...
# ---------------------------------------------------------
doTestGmailPackage(){

   do_log "DEBUG START doTestGmailPackage"
	
   cat doc/txt/qto/tests/pckg/gmail-package.test.txt
   sleep "$sleep_interval"

   bash src/bash/qto/qto.sh -a create-full-package -a gmail-package
   sleep "$sleep_interval"
   source $PRODUCT_DIR/lib/bash/funcs/flush-screen.sh
   do_flush_screen

   do_log "DEBUG STOP  doTestGmailPackage"
}
# eof func doTestGmailPackage


# eof file: src/bash/qto/funcs/gmail-package.test.sh
<file_sep>---- DROP TABLE IF EXISTS imgs CASCADE;

SELECT 'create the "imgs" table'
; 
   CREATE TABLE imgs (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , name           varchar (200) NOT NULL DEFAULT 'image figure title ...'
    , relative_path  varchar (1000) NOT NULL DEFAULT 'src/perl/qto/public/dat/img/qto/...'
    , http_path      varchar (4000) NOT NULL DEFAULT 'https://raw.githubusercontent.com/YordanGeorgiev/qto/dev/doc/img/..'
    , style          varchar (100) NOT NULL DEFAULT 'width: 800px; height: 600x'
    , item_guid      UUID NOT NULL DEFAULT gen_random_uuid()
    , description    varchar (4000) NOT NULL DEFAULT 'txt to appear on alt ...' 
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_imgs_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

   create unique index idx_uniq_imgs_id on imgs (id);



SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.imgs'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 

--The trigger:
   CREATE TRIGGER trg_set_update_time_on_imgs 
   BEFORE UPDATE ON imgs 
   FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'imgs'::regclass;

<file_sep># file: src/bash/qto/funcs/increase-date.func.sh
# today is --date="+1 day"
# tommorrow is --date="+1 day"
#
# ---------------------------------------------------------
# cat doc/txt/qto/funcs/increase-date.func.txt
# ---------------------------------------------------------
do_increase_date(){

   test -z "${PROJ_INSTANCE_DIR:-}" && PROJ_INSTANCE_DIR="$PRODUCT_DIR"
   #
   do_export_json_section_vars $PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json '.env.db'
   mix_data_dir=$PROJ_INSTANCE_DIR/dat/mix

   # find the latest project_daily_txt_dir
   latest_proj_daily_dir=""
   latest_proj_daily_dir=$(find $mix_data_dir -type d -maxdepth 3|grep -v txt|grep -v json|grep -v xls|sort -nr|head -1)
   echo "using the following latest_proj_daily_dir: $latest_proj_daily_dir"
   sleep 1

	# debug set -x
   if [[ ${tgt_date+x} && -n $tgt_date ]] ; then

		case "$tgt_date" in
			  today)
         		export tgt_date=$(date --date="+0 day" "+%Y-%m-%d")
					;;
			  tomorrow)
         		export tgt_date=$(date --date="+1 day" "+%Y-%m-%d")
					;;
			  yesterday)
         		export tgt_date=$(date --date="-1 day" "+%Y-%m-%d")
					;;
			  *)
					# check that the date is the %Y-%m-%d format
				 	doRunCmdOrExit 'date +%Y-%m-%d -d '"$tgt_date"
		esac

   else
      export tgt_date=$(date --date="+0 day" "+%Y-%m-%d")
   fi

	msg="using the following date: \"$tgt_date\""
  	do_log "INFO $msg"

   # define the today's daily_data_dir
   tgt_days_monthly_data_dir="$mix_data_dir"/$(date "+%Y" -d "$tgt_date")
   tgt_days_monthly_data_dir="$tgt_days_monthly_data_dir"/$(date "+%Y-%m" -d "$tgt_date")
   mkdir -p $tgt_days_monthly_data_dir
   export daily_data_dir="$tgt_days_monthly_data_dir"'/'$(date "+%Y-%m-%d" -d "$tgt_date")

   error_msg="
   nothing can be done - as the daily data dir \$daily_data_dir :
      $daily_data_dir
   already exists !!!
      "
   test -d "$daily_data_dir" && do_exit 1 "$error_msg"

   todays_tmp_dir=$tmp_dir/$(date "+%Y-%m-%d" -d "$tgt_date")    # becauses of vboxsf !!!
   cmd="cp -vr $latest_proj_daily_dir $todays_tmp_dir/"

   doRunCmdOrExit "$cmd" ; cd $todays_tmp_dir

   # foreach sh or txt file
   while read -r f ; do
      export last_day=$(echo $f|cut -d'.' -f 4);

      table=$(echo $f|cut -d'.' -f 3); proj=$(echo $f|cut -d'.' -f 2);
      #debug do_log "DEBUG table: $table"

      file_ext=$(echo $f|cut -d'.' -f 5);
      # debug do_log "DEBUG file_ext: $file_ext"

      mv "$f" '.'"$proj"."$table".`date "+%Y-%m-%d" -d "$tgt_date"`."$file_ext"
   # obs works only on gnu find !
   done < <(find . -type f -regex ".*\.\(sh\|txt\)")


   # search and replace the daily
   today=$(date +%Y-%m-%d)
   while read -r f ; do
      perl -pi -e 's/\@'"$last_day"'/\@'"$today"'/g;' "$f" ;
   done < <(find . -type f -name '*.txt' -o -name '*.sh')

   rm -f *.bak       # remove any possible bak files
   mv $todays_tmp_dir $daily_data_dir
   rm -rfvd $daily_data_dir/tmp

   export daily_data_dir=$mix_data_dir/$(date "+%Y")/$(date "+%Y-%m")/$(date "+%Y-%m-%d")

   # remove all the tmp xls files
   rm -v "$daily_data_dir/xls/"'/~'*.xlsx

   # foreach xls file
   while read -r f ; do
      file_name="${f##*/}"
      proj=$(echo $file_name|cut -d'.' -f 1);
      table=$(echo $file_name|cut -d'.' -f 2);
      timestamp=$(echo $file_name|cut -d'.' -f 3);
      # file_name="${f%.*}"
      # file_name=$(basename -- "$f")
      file_ext="${f##*.}"
      mkdir -p "$daily_data_dir/xls"
      do_log "DEBUG" "mv $f" -> "$daily_data_dir/$proj"."$table".$(date "+%Y%m%d_%H%M%S" -d "$tgt_date")."$file_ext"
      # and I just don't know where this one comes from, but it is fake !!!
      rm -v "$daily_data_dir/$proj"."$table".$(date "+%Y%m%d_%H%M%S" -d "$tgt_date")."$file_ext"
      mv "$f" "$daily_data_dir/xls/$proj"."$table".$(date "+%Y%m%d_%H%M%S" -d "$tgt_date")."$file_ext"
   done < <(find $daily_data_dir/xls -type f \( -name "*.xlsx" -o -name "*.xls" \))

   doBackupPostgresDb

   msg=" OK for creating the daily project dir:
         $daily_data_dir"
   do_log "INFO ""$msg"

	do_log "DEBUG STOP  doIncreaseDate"
}

# eof file: src/bash/qto/funcs/increase-date.func.sh
<file_sep>#!/usr/bin/env bash


main(){
   do_run_test_01 "test-01 the usage is print if no \$1 and \$2 are passed"
   do_run_test_02 "test-02 the usage is print if no \$2 are passed"
   do_run_test_03 "test-03 a normal run when both \$1 and \$2 are passed"
}


do_run_test_01(){
  echo $1 ; shift 
  to_tst='psql-code-generator - a script for generating code'
  out=`echo $(bash src/tpl/psql-code-generator/psql-code-generator.sh)|grep -c "$to_tst"`
  test $out -eq 1 && echo     test-01 ok
  test $out -eq 0 && echo     test-01 failed
  echo -e "\n\n"
  sleep 1
  do_flush_screen
}


do_run_test_02(){
  echo $1 ; shift 
  to_tst='psql-code-generator - a script for generating code'
  out=`echo $(bash src/tpl/psql-code-generator/psql-code-generator.sh dev_qto)|grep -c "$to_tst"`
  test $out -eq 1 && echo     test-02 ok
  test $out -eq 0 && echo     test-02 failed
  echo -e "\n\n"
  sleep 1
  do_flush_screen
}


do_run_test_03(){
  echo $1 ; shift 
  bash src/tpl/psql-code-generator/psql-code-generator.sh dev_qto release_issues monthly_issues
  sleep 1
  # do_flush_screen
}


main
<file_sep>#!/usr/bin/env bash


# v1.1.4
#------------------------------------------------------------------------------
# creates a package from the relative file paths specified in the .dev file
#------------------------------------------------------------------------------
doTestCreateRelativePackage(){
	do_log " START : create-relative-package.test"

	doSpecCreateRelativePackage

	doHelpCreateRelativePackage
   
   #set -x
	# test the call with the include file of the current env - usually dev
   exit_code=0
	bash src/bash/qto/qto.sh -a create-relative-package
   export exit_code=$?
	do_log " create-relative-package.test-1 exit_code: $exit_code "
   sleep "$sleep_interval"
   test $exit_code -ne 0 && return
	
	# test the call with the include file of the tst env
   bash src/bash/qto/qto.sh -a create-relative-package -i met/.tst.qto
   export exit_code=$?
	do_log " create-relative-package.test-2 exit_code: $exit_code "
   sleep "$sleep_interval"
   test $exit_code -ne 0 && return
	
	# test the call with the include file of the prd env
	bash src/bash/qto/qto.sh -a create-relative-package -i met/.prd.qto
   export exit_code=$?
	do_log " create-relative-package.test-3 exit_code: $exit_code "
   sleep "$sleep_interval"
   test $exit_code -ne 0 && return
	
   # test the call with the include file of the prd env
   bash src/bash/qto/qto.sh -a create-relative-package -i met/.git.qto
   export exit_code=$?
	do_log " create-relative-package.test-4 exit_code: $exit_code "
   sleep "$sleep_interval"
   test $exit_code -ne 0 && return
	
	do_log " STOP  : create-relative-package.test"
}
#eof doCreateRelativePackage
<file_sep># file: doc/cheats/qto-cheat-sheet.sh

psql -d "$postgres_app_db" -c "GRANT SELECT,INSERT,UPDATE,DELETE,TRUNCATE ON ALL TABLES IN SCHEMA public TO $postgres_app_usr; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO $postgres_app_usr; GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO $postgres_app_usr; ALTER DEFAULT PRIVILEGES FOR USER $postgres_app_usr IN SCHEMA public GRANT SELECT ON TABLES TO    $postgres_app_usr;  "


# how-to check the perl syntax of single perl file : 
perl -MCarp::Always -I `pwd`/src/perl/*/ -I `pwd`/src/perl/*/lib/ -wc \
   src/perl/qto/lib/Qto/App/Db/In/Postgres/RdrPostgresDb.pm

# start load env vars
export PROJ_INSTANCE_DIR=`pwd`  # or could be any qto proj
export PROJ_CONF_FILE=`pwd`/cnf/env/prd.env.json  # or could be any qto proj's conf file
source lib/bash/funcs/export-json-section-vars.sh
do_export_json_section_vars $PROJ_INSTANCE_DIR/cnf/env/prd.env.json '.env.db'
alias psql="PGPASSWORD=${postgres_sys_usr_admin_pw:-} psql -v -t -X -w -U ${postgres_sys_usr_admin:-} --host $postgres_rdbms_host --port $postgres_rdbms_port"

./src/bash/qto/qto.sh -a provision-db-admin # change the db-admin pw
./src/bash/qto/qto.sh -a provision-app-user # change the app user pw

# how-to load xls sheets into tables , obs - do_truncate_tables=1 WILL truncate the postgres tables
export do_truncate_tables=0 ; clear ; perl src/perl/qto/script/qto.pl --do xls-to-db --tables system_guide_doc

clear ; ./src/bash/qto/qto.sh -a backup-postgres-db-inserts
clear ; ./src/bash/qto/qto.sh -a backup-postgres-table -t readme_doc

export docs_root_dir=`pwd` # or could be any qto proj dir
clear ; ./src/bash/qto/qto.sh -a generate-md-docs  # export the documentation of the desired db
clear ; ./src/bash/qto/qto.sh -a generate-pdf-docs  # export the documentation of the desired db

./src/bash/qto/qto.sh -a mojo-hypnotoad-start # start the hypnotoad web server

# how-to run sql as the postgres user
clear ; sudo su - postgres -c "psql -d postgres -c \"drop database dev_qto\""
clear ; sudo su - postgres -c "psql -d postgres -c \"create database dev_qto\""

# get the id,guid,name from the back-end api
mojo get 'http://host-name:8082/tst_qto/hiselect/requirements_doc?bid=0' |jq '.dat[]|.id,.guid,.name'
mojo get 'http://host-name:8082/tst_qto/hiselect/requirements_doc?bid=0&oa=seq' |jq '.dat[]|{id,guid,name} '

psql -t -d dev_qto -c "select array_to_json(array_agg(row_to_json(t))) from ( select * from release_issues ) t"|jq


# list all the tables in the terminal
clear ; psql -d dev_qto -t -c '\dt' | cut -c 11- | perl -ne 's/^([a-z_0-9]*)( )(.*)/$1/; print '

# how-to backup dirs from remote to local

export ssh_user=ubuntu; export ssh_server=172.16.31.10
export tgt_dir=/tmp/dir-to-be-overwritten; export src_dir=/tmp/dir-to-be-rsynced
rsync -e "ssh -l USERID -i ~/.ssh/id_rsa.tst.qto" -av -r --partial --progress --human-readable \
   --stats --delete-excluded $ssh_user@$ssh_server:$src_dir $tgt_dir

export ssh_server=qto.fi
export ssh_user=ubuntu
export src_dir=/home/ubuntu/opt/qto
export tgt_dir=/hos/opt/opt
rsync -e "ssh -l USERID -i ~/.ssh/id_rsa.aws-ec2.qto.prd" -av -r --partial --progress
   --human-readable --stats $ssh_user@$ssh_server:$src_dir $tgt_dir

# eof file: doc/cheats/qto-cheat-sheet.sh
<file_sep>doCheckTmuxIsInstalled(){
   which tmux >/dev/null 2>&1 ||
      { do_log "FATAL ERROR - tmux is not installed
         or not in PATH. To install it do run
         sudo apt-get install -y tmux
         or
         sudo yum install -y tmux
         or
         brew install tmux
         Aborting." >&2; exit 1; }
}


terminal_size(){
  stty size 2>/dev/null | awk '{ printf "-x%d -y%d", $2, $1 }'
}

session_exists(){
  tmux has-session -t "$1" 2>/dev/null
}

add_window(){
  tmux new-window -d -t "$1:" -n "$2" -c "$3"
}


new_tmux_session(){
  cd "$3" &&
  tmux new-session -d -s "$1" -n "$2" $4
}
<file_sep>#  QTO APPLICATION CONCEPTS
* [1. INTRO](#1-intro)
  * [1.1. PURPOSE](#11-purpose)
  * [1.2. AUDIENCE](#12-audience)
  * [1.3. RELATED DOCUMENTATION](#13-related-documentation)
* [2. BUSINESS LOGIC](#2-business-logic)
  * [2.1. PROJECTS MANAGEMENT](#21-projects-management)
  * [2.2. INCIDENTS AND WORK MANAGEMENT VIA ISSUES](#22-incidents-and-work-management-via-issues)
    * [2.2.1. Monthly issues](#221-monthly-issues)
    * [2.2.2. Yearly_issues](#222-yearly_issues)
    * [2.2.3. Categories](#223-categories)
    * [2.2.4. Issues status](#224-issues-status)
    * [2.2.5. Issues management via time intervals](#225-issues-management-via-time-intervals)
  * [2.3. QUESTIONS](#23-questions)
  * [2.4. IDEAS](#24-ideas)
  * [2.5. PROBLEMS](#25-problems)
* [3. DEFINITIONS](#3-definitions)
  * [3.1. RELEASE](#31-release)
* [4. DOCUMENTATION](#4-documentation)
  * [4.1. THE CONCEPTS DOC ](#41-the-concepts-doc-)
  * [4.2. THE USERSTORIES DOC](#42-the-userstories-doc)
  * [4.3. THE REQUIREMENTS DOCUMENT](#43-the-requirements-document)
  * [4.4. THE SYSTEM GUIDE](#44-the-system-guide)
  * [4.5. THE DEVOPS GUIDE](#45-the-devops-guide)
  * [4.6. THE MAINTENANCE AND OPERATIONS GUIDE](#46-the-maintenance-and-operations-guide)




    

## 1. INTRO


    

### 1.1. Purpose
The purpose of this document is to present the concepts and the business logic of the qto application for this current version.

    

### 1.2. Audience
This document could be of interest for any potential and actual users of the application. Key-users must read, understand and even be able to present and explain the contents of this document. 
Developers and Architects working on the application MUST read and understand this document at least to the extent of their own contribution for the application. 

    

### 1.3. Related documentation
This document is part of the qto application documentation-set, which contains the following documents:
 - readme_doc-0 - the initial landing readme doc for the project
 - userstories_doc-0 - the collection of user-stories used to describe "what is desired"
 - requirements_doc-0 - the structured collection of the requirements 
 - system_guide_doc-0 - architecture and System description
 - devops_guide_doc-0 - a guide for the developers and DevOps operators
 - installations_doc-0 - a guide for installation of the application
 - enduser_guide_doc-0 - the guide for the usage of the UI ( mainly ) for the end-users
 - concepts_doc-0 - the concepts doc 

All the documents should be updated and redistributed in combination of the current version of the application and should be found under the doc/MD directory.


    

## 2. BUSINESS LOGIC


    

### 2.1. Projects management
Any undertaking or endeavour, carried out individually or collaboratively and possibly involving research or design, that is carefully planned, usually by a project team, to achieve a particular aim could be considered a project. The main purpose of the QTO application is to manage one or more projects from a single User Interface.


    

### 2.2. Incidents and work management via issues
Issue item is the shortest possible description of task, activity, note or anything requiring distinguishable and preferably measurable action or producing verifiable outcome.
Issues could be of different types - tasks, activities, notes etc. 
Simply said the issues are the split by deliverable items of work to be done by the project team.
Each issue MUST BE assigned to one and only one person, to foster responsibility and accountability. 
The issues are split into different periods for greater flexibility enabling both historical storage and tracking AND susceptibility for change.

    

#### 2.2.1. Monthly issues
The issues in qto are stored and reviewed on per monthly basis , so that the issues in of the January 2020 are stored in the monthly_issues_202001 table, the issues of February 2020 are stored in the monthly_issues_202002 etc. 
At the end of each month ( and  often also in the middle of the period ) the issues are upserted ( that is new issues are created and those with existing ids updated ) into the yearly_issues_2020 table.


    

#### 2.2.2. Yearly_issues
The yearly_issues table MUST have the same structure as the currently active monthly_issues table
In SCRUM the backlog is usually reviewed every 2 weeks, you could quickly copy the monthly_issues table and star t using with sprint_01 , sprint_02 , sprint_&lt;&lt;n&gt;&gt; tables ...

    

#### 2.2.3. Categories
Each issue item could be categorised under one and only one category. One category might have 1 or more issues. 
The issues can be split into different categories. You can define your categories from the https://qto.fi/qto/list/category page.

    

#### 2.2.4. Issues status
Each issue could be in a different status. You can define the different statuses for an issue from the issue_status list page : https://qto.fi/qto/list/issues_status

    

#### 2.2.5. Issues management via time intervals
The issues are basically organised into the following time intervals:
 - release_issues - the issues to be handled till the next release ( usually, but not always a 2 weeks period)
 - monthly_issues - the issues to be handled during the next month
 - yearly - the issues to be handled during the next year
 - quinquennially
 - decadally

So that in the end of each previous time period you could go trough the issues of that period and transfer up and down in the time scale.
The product instance owners have an automated way of transferring the issues from one table to another described in the maintenance_guide_doc-200111060442 document.

    

### 2.3. Questions
Sometimes during the workings of your project you encounter problems, which are complex enough not to allows the definition of an issue. In those cases it would be more rational to just register a question, a problem, an idea, discuss it or review it later on, and define the issue as soon as the problem domain is understood and even possible issue solution could be proposed.
Different question statuses can be defined.

    

### 2.4. Ideas
Your organisation might collect, sort and evaluate ideas so that they could be later one used as the row data for issues ( aka concrete work descriptions to be followed).
Different ideas statuses can be defined.

    

### 2.5. Problems
Sometimes during the workings of your project you encounter problems, which are complex enough not to allows the definition of an issue. In those cases it would be more rational to just register a question, a problem, an idea, discuss it or review it later on, and define the issue as soon as the problem domain is understood and even possible issue solution could be proposed.
Quite often the row input material for the issues are the problems encountered - the better you collect, describe and prioritise the list of problems to tackle the better you will be able to organise the issues to be completed. 
Different problems statuses can be defined.

    

## 3. DEFINITIONS
This section contains definitions of terms within the context of the qto application.

    

### 3.1. Release
A qto release is the artefact you can download from the following GitHub page:
https://github.com/YordanGeorgiev/qto/releases.

    

## 4. DOCUMENTATION


    

### 4.1. The Concepts doc 
This document to contain different planned and/or possible concepts in order to ease your understanding and further exploitation of the application. Of course, we, as QTO team, are here to support your organisation while internal development is on-going. In case the day-to-day requirements extends and/or change, you will be able to make these amendments independently if you will.

    

### 4.2. The UserStories doc
The user stories doc contains the structurized and formal description of the user stories for the application.

    

### 4.3. The Requirements document
The requirements document stores all the requirements for the qto application.

    

### 4.4. The System Guide
The System Guide describes the current and factual state of the System you are actually using.

    

### 4.5. The DevOps Guide
The DevOps Guide contains information on how-to develop, operate and maintain the Application from a DevOps / developer perspective.

    

### 4.6. The Maintenance and Operations Guide
The Maintenance and Operations Guide contains information on how-to maintain and operate the Application.

    

<file_sep>
do_chk_install_postgres(){

   postgres_repo_url='http://apt.postgresql.org/pub/repos/apt/'
   postgres_repo_fle='/etc/apt/sources.list.d/pgdg.list'

	which psql 2>/dev/null || {
		sudo sh -c 'echo "deb '"$postgres_repo_url"' `lsb_release -cs`-pgdg main" >> '"$postgres_repo_fle"
		wget -q 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' -O - | sudo apt-key add -
		sudo apt-get update

      packages="$(cat << EOF_PACKAGES
         postgresql
         postgresql-contrib
         libpq-dev
         pgadmin3
EOF_PACKAGES
)"
      
      while read -r package ; do 
         sudo apt-get install -y $package 
      done < <(echo "$packages");
	}

	which psql 2>/dev/null || {
		cat << EOF_UNINSTALL
      FATAL failed to install postgres !!! You might have to run those manually
      sudo /etc/init.d/postgresql stop
      sudo /etc/init.d/postgresql status
      sudo dpkg --configure -a
      sudo rm -rf /var/log/postgresql/
      sudo rm -rf /var/lib/postgresql/
      sudo apt-get purge -y remove pgdg-keyring postgresql*
      sudo apt-get -y purge postgresql
      sudo apt-get -y purge postgresql-contrib
      sudo apt-get -y purge libpq-dev
      sudo apt-get -y purge pgadmin3
      sudo apt-get purge -y postgresql-client
      sudo apt-get -y purge psql
EOF_UNINSTALL
      exit 1
	}

	echo "check the postgres version:"
	sudo -u postgres psql postgres -c "SELECT version();" | grep 'PostgreSQL'

   ## ensure the postresql starts on boot 
   sudo update-rc.d postgresql enable

}
<file_sep>#!/usr/bin/env bash 

#v1.1.0
#------------------------------------------------------------------------------
# tests the full package creation
#------------------------------------------------------------------------------
doTestCreateFullPackage(){
	cd $PRODUCT_DIR
	do_log " INFO START : create-full-package.test"
	
	cat doc/txt/qto/tests/pckg/create-full-package.test.txt

	doSpecCreateFullPackage

	doHelpCreateFullPackage

   export exit_code=0
	bash src/bash/qto/qto.sh -a create-full-package
   export exit_code=$?
	do_log " create-relative-package.test-1 exit_code: $exit_code "
   sleep "$sleep_interval"
   test $exit_code -ne 0 && return

	bash src/bash/qto/qto.sh -a create-full-package -i $PRODUCT_DIR/met/.tst.qto
   export exit_code=$?
	do_log " create-relative-package.test-1 exit_code: $exit_code "
   sleep "$sleep_interval"
   test $exit_code -ne 0 && return
	
	bash src/bash/qto/qto.sh -a create-full-package -i $PRODUCT_DIR/met/.prd.qto
   export exit_code=$?
	do_log " create-relative-package.test-1 exit_code: $exit_code "
   sleep "$sleep_interval"
   test $exit_code -ne 0 && return
	
	bash src/bash/qto/qto.sh -a create-full-package -i $PRODUCT_DIR/met/.git.qto
   export exit_code=$?
	do_log " create-relative-package.test-1 exit_code: $exit_code "
   sleep "$sleep_interval"
   test $exit_code -ne 0 && return
	

	do_log " INFO STOP  : create-full-package.test"
}
#eof test doCreateFullPackage
<file_sep>#
# Informs the user that DB provisioning is done
# and that they can start the Mojo server now.
#
# Pops up in the end of execution of DB provisioning:
# ./src/bash/qto/qto.sh -a provision-db-admin -a run-qto-db-ddl -a load-db-data-from-s3 -a notify-success
# ==
# ./src/bash/qto/qto.sh -a set-roles -a create-db -a fill-db -a success
# 
doNotifyProvisioningSuccess(){
   
   cat << EOF_FINAL_SUCCESS
   :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
      QTO installation completed succesfully.

      Please run this command to start the web server:

      ./3-start-server.sh
   :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
EOF_FINAL_SUCCESS

}
<file_sep>#  REQUIREMENTS
* [1. INTRO](#1-intro)
  * [1.1. AUDIENCE](#11-audience)
  * [1.2. RELATED DOCUMENTATION](#12-related-documentation)
* [2. DEPLOYABILITY](#2-deployability)
  * [2.1. DEVOPS DEPLOYABILITY NO LONGER THAN A WEEK RELEASE CYCLE](#21-devops-deployability-no-longer-than-a-week-release-cycle)
  * [2.2. AUTOMATED AWS DEPLOYMENT IN LESS THAN AN HOUR](#22-automated-aws-deployment-in-less-than-an-hour)
  * [2.3. FULL DEPLOYMENT IN LESS THAN AN HOUR](#23-full-deployment-in-less-than-an-hour)
    * [2.3.1. A working instance deployment by simple unzip command](#231-a-working-instance-deployment-by-simple-unzip-command)
    * [2.3.2. Binary prerequisites check script](#232-binary-prerequisites-check-script)
    * [2.3.3. Required Perl modules installation](#233-required-perl-modules-installation)
    * [2.3.4. Installation documentation](#234-installation-documentation)
  * [2.4. FULL APPLICATION CLONE CREATION IN 5MIN](#24-full-application-clone-creation-in-5min)
    * [2.4.1. Single shell call for postgres db creation and initial data load](#241-single-shell-call-for-postgres-db-creation-and-initial-data-load)
    * [2.4.2. One liner for single restore for both full db and inserts only](#242-one-liner-for-single-restore-for-both-full-db-and-inserts-only)
  * [2.5. ONE AND ONLY ONE PLACE FOR CONFIGURATION ](#25-one-and-only-one-place-for-configuration-)
* [3. RELIABILITY AND STABILITY](#3-reliability-and-stability)
  * [3.1. ZERO TOLLERANCE TOWARDS CRASHING](#31-zero-tollerance-towards-crashing)
  * [3.2. ZERO TOLERANCE TOWARDS BUGS](#32-zero-tolerance-towards-bugs)
  * [3.3. E2E VERTICALITY](#33-e2e-verticality)
  * [3.4. DAILY BACKUPS](#34-daily-backups)
  * [3.5. LOGGING](#35-logging)
  * [3.6. FULL BACKUP TO THE CLOUD IN LESS THAN 5 MINUTES](#36-full-backup-to-the-cloud-in-less-than-5-minutes)
* [4. USABILITY](#4-usability)
  * [4.1. APPLICATION WIDE TOP-BAR USABILITY](#41-application-wide-top-bar-usability)
  * [4.2. APPLICATION WIDE LEFT MENU USABILITY](#42-application-wide-left-menu-usability)
    * [4.2.1. Left menu global configurability](#421-left-menu-global-configurability)
    * [4.2.2. Left menu personal customisability](#422-left-menu-personal-customisability)
  * [4.3. UI USABILITY](#43-ui-usability)
    * [4.3.1. Application wide top-bar usability](#431-application-wide-top-bar-usability)
    * [4.3.2. Login page usability](#432-login-page-usability)
    * [4.3.3. Landing / home page usability](#433-landing-/-home-page-usability)
    * [4.3.4. View page usability](#434-view-page-usability)
  * [4.4. ONELINER SHELL CALLS](#44-oneliner-shell-calls)
    * [4.4.1. Database recreation and DDL scripts run one-liners](#441-database-recreation-and-ddl-scripts-run-one-liners)
    * [4.4.2. Table(s) load via aa single one-liner](#442-table(s)-load-via-aa-single-one-liner)
* [5. SCALABILITY](#5-scalability)
  * [5.1. HORIZONTAL PERFORMANCE SCALABILITY](#51-horizontal-performance-scalability)
  * [5.2. FEATURE SCALABILITY](#52-feature-scalability)
  * [5.3. SETUP SCALABILITY](#53-setup-scalability)
  * [5.4. PROJECTS DATABASES SCALABILITY](#54-projects-databases-scalability)
  * [5.5. QTO APPLICATION CLONES SCALABILITY](#55-qto-application-clones-scalability)
* [6. PERFORMANCE](#6-performance)
  * [6.1. PAGE LOAD MAXIMUM TIME](#61-page-load-maximum-time)
  * [6.2. LOGIN, LOGOUT](#62-login-logout)
* [7. MULTI-INSTANCE OPERABILITY AND DEPLOYABILITY](#7-multi-instance-operability-and-deployability)
  * [7.1. ENVIRONMENT TYPE SELF-AWARENESS](#71-environment-type-self-awareness)
  * [7.2. CROSS RUNNING BETWEEN INSTANCES OF DIFFERENT TYPES](#72-cross-running-between-instances-of-different-types)
* [8. UI REQUIREMENTS](#8-ui-requirements)
  * [8.1. CRUDS](#81-cruds)
    * [8.1.1. Execution time](#811-execution-time)
    * [8.1.2. Visual indication](#812-visual-indication)
  * [8.2. CLARITY ON ERRORS](#82-clarity-on-errors)
  * [8.3. INFORMATION SEARCH](#83-information-search)
    * [8.3.1. Global project search](#831-global-project-search)
    * [8.3.2. Quick search per page](#832-quick-search-per-page)
  * [8.4. LOGIN PAGE REQUIREMENTS](#84-login-page-requirements)
  * [8.5. LIST PAGE REQUIREMENTS](#85-list-page-requirements)
    * [8.5.1. View page requirements](#851-view-page-requirements)
* [9. INTEROPERABILITY](#9-interoperability)
  * [9.1. DATA IMPORT](#91-data-import)
    * [9.1.1. Import from xls](#911-import-from-xls)
    * [9.1.2. Import from json format](#912-import-from-json-format)
    * [9.1.3. Files upload ](#913-files-upload-)
  * [9.2. DATA EXPORT](#92-data-export)
    * [9.2.1. Export to pdf](#921-export-to-pdf)
    * [9.2.2. Export to md](#922-export-to-md)
    * [9.2.3. Export to xls](#923-export-to-xls)
    * [9.2.4. Export to Microsoft docx](#924-export-to-microsoft-docx)
    * [9.2.5. Export to json](#925-export-to-json)
* [10. SECURITY](#10-security)
  * [10.1. AUTHENTICATION](#101-authentication)
    * [10.1.1. QTO_NO_AUTH mode](#1011-qto_no_auth-mode)
    * [10.1.2. Simple Native authentication mode](#1012-simple-native-authentication-mode)
      * [10.1.2.1. User email and password matching for login success](#10121-user-email-and-password-matching-for-login-success)
      * [10.1.2.2. The passwords must NOT be stored in clear text](#10122-the-passwords-must-not-be-stored-in-clear-text)
      * [10.1.2.3. name ...](#10123-name-)
      * [10.1.2.4. Passwords sensibility](#10124-passwords-sensibility)
    * [10.1.3. RBAC authentication mode via JWT](#1013-rbac-authentication-mode-via-jwt)
      * [10.1.3.1. Login](#10131-login)
      * [10.1.3.2. Must be stateless and thus horizontally scalable](#10132-must-be-stateless-and-thus-horizontally-scalable)
      * [10.1.3.3. Must not use permanent cookies](#10133-must-not-use-permanent-cookies)
      * [10.1.3.4. Must not use localStore, but Authorisation header](#10134-must-not-use-localstore-but-authorisation-header)
      * [10.1.3.5. Must have arymetric signature](#10135-must-have-arymetric-signature)
      * [10.1.3.6. The tokens must be self-contained](#10136-the-tokens-must-be-self-contained)
      * [10.1.3.7. Tokens expiration](#10137-tokens-expiration)
      * [10.1.3.8. Must support whitelisting](#10138-must-support-whitelisting)
      * [10.1.3.9. Must support black-listing](#10139-must-support-black-listing)
  * [10.2. AUTHORISATION](#102-authorisation)
    * [10.2.1. QTO_NO_AUTH mode authorisation](#1021-qto_no_auth-mode-authorisation)
    * [10.2.2. Simple Native mode authorisation](#1022-simple-native-mode-authorisation)
    * [10.2.3. RBAC mode authorisation](#1023-rbac-mode-authorisation)
      * [10.2.3.1. Role-based Access Control](#10231-role-based-access-control)
      * [10.2.3.2. Authorisation via Json Web Tokens during login](#10232-authorisation-via-json-web-tokens-during-login)
      * [10.2.3.3. Roles based Access Control list for routes and items](#10233-roles-based-access-control-list-for-routes-and-items)
      * [10.2.3.4. Role Based Access Control on tables row level](#10234-role-based-access-control-on-tables-row-level)
      * [10.2.3.5. Support for segregation of duties](#10235-support-for-segregation-of-duties)
    * [10.2.4. Role level based security ](#1024-role-level-based-security-)
* [11. DOCUMENTATION](#11-documentation)
  * [11.1. DOCUMENTATION COMPLETENESS](#111-documentation-completeness)
  * [11.2. DOCUMENTATION AND CODE BASE SYNCHRONIZATION](#112-documentation-and-code-base-synchronization)
    * [11.2.1. Requirements push](#1121-requirements-push)




    

## 1. INTRO


    

### 1.1. Audience
This document is aimed for any potential and actual developers of the qto application. Product Owners, Developers and Architects working on the application MUST read and understand this document at least to the extend of their own contribution for the application. 

    

### 1.2. Related documentation
This document is part of the QTO application documentation-set, which contains the following documents:
 - ReadMe - the initial landing readme doc for the project
 - UserStories - the collection of user-stories used to describe "what is desired"
 - End-User Guide - the guide for the usage of the UI ( mainly ) for the end-users
 - Requirements - the structured collection of the requirements 
 - SystemGuide - architecture and System description
 - DevOps Guide - a guide for the developers and devops operators
 - Installation Guide - a guide for installation of the application
 - Concepts - the concepts doc 

You can access all the latest  qto documentation from qto site: 
https://qto.fi
in it's native format.
All the documents are updated and redistributed in combination of the current version of the application in both md and pdf file format and can be found under the following directories:
 - doc/md
 - doc/pdf

    

## 2. DEPLOYABILITY
The qto must be easily deployable on any Unix like OS. 
Windows family based OS'es are explicitly out of the scope of the qto tool. Any qto instance should be configurable as easily as possible for its version.

    

### 2.1. DevOps deployability no longer than a week release cycle
The qto system should provide the CI  infrastructure for the capability to perform frequent releases, with release cycles no longer than a week. For larger releases 2 weeks cycle are acceptable too, with no more than 3 larger releases in a row.

    

### 2.2. Automated AWS deployment in less than an hour
The qto system should be automatically deployable to aws in less than an hour, so that the deployment operator should execute no more than 5 documented commands.

    

### 2.3. Full deployment in less than an hour
The full System should be ready for use by end-users in the latest Ubuntu LTE OS in less than an hour. The whole deployment process MUST BE as automated as possible.

    

#### 2.3.1. A working instance deployment by simple unzip command
The qto tool could be deployed by a simply unzip of the full package into a host having the proper binary configuration, which must have all of the documentation and scripts to provide assistance for the setup and the configuration of the tool as well as the initial data to populate the qto database. 

    

#### 2.3.2. Binary prerequisites check script
All the binaries which are required for the running of the tool must be checked by a user-friendly binaries prerequisites check script. 


    

#### 2.3.3. Required Perl modules installation
All the required Perl modules must be part of the deployer script. The Perl modules should be installed as non-root user having sudo, which could be removed during the start-up of the application for security reasons.

    

#### 2.3.4. Installation documentation
The installation of the required Postgres db must be documented in the DevOps guide, which should have the markdown version in the doc directory of the deployment package.

    

### 2.4. Full application clone creation in 5min
A DevOps operator should be able to perform an application clone ( having app-name changed to new-app-name etc. ) of the Qto application in less than 5 minutes. 

    

#### 2.4.1. Single shell call for postgres db creation and initial data load
The creation of the Postgres database of a qto project should be doable via a single shell call. 

    

#### 2.4.2. One liner for single restore for both full db and inserts only
The full database should be loadable form a db dump either from a full dump or from the db-inserts dump only.

    

### 2.5. One and only one place for configuration 
Whenever there is a configuration entry indicating part of the configuration of an application instance it should be stored in 1 and only 1 place in the configuration file of the instance, so that any person not familiar with the internal logic of the application could find all the configuration entries in one and only one place including the secrets.

    

## 3. RELIABILITY AND STABILITY


    

### 3.1. Zero tollerance towards crashing
Crashing in normally configured and operating environment must not be tolerated, as soon as any crash has occurred a bug must be registered and the bug set with the highest possible prio towards the features pipeline. 

    

### 3.2. Zero tolerance towards bugs
All bugs and inconsistencies must be dealt with top priority by passing new features implementation.  Should the average amount of bugs increase after a release a purely bug fixing release should follow. Any reasonable refactoring could and should bypass the new features implementation. tolerance

    

### 3.3. e2e verticality
The qto source code artefact must contain both the documentation and the references for the published and running qto application AND data, so that any potential developer or user of the application could easily setup a new instance or simply use one relying on the fact that the full stack has been tested and applied according to that very latest release he or she is using.

    

### 3.4. Daily backups
The daily backups of all instance project databases data and secrets should be performed automatically as indispensable part of the functioning of the application. 

    

### 3.5. Logging
The application should support configurable logging to STDOUT and STDERR for the following levels - debug, info, warn, trace. 

    

### 3.6. Full backup to the cloud in less than 5 minutes
A full backup of the software, configuration and data for the qto and/or another project database should be doable in less than 5 minutes. The backup should be easily searchable from the cloud as well. 

    

## 4. USABILITY
The interaction with each endpoint and interface of an application instance should be as user-friendly as possible. 
As abstract as it may sound the tool must be multi-dimensionally and vertically integrated regarding the questions what, how and why towards a new person interacting with the tool by the usage of code comments , links from the documentations and uuids to be used for simple greping from the docs till the source code. 

    

### 4.1. Application wide top-bar usability
The top-bar of the application must be ALWAYS selected during a page load and particularly the omni-search box, so that the user could straight way start quick filtering the textual content of ANY page by typing at least 3 chars in the search box.
The top-bar must be visible on ALL pages, except those aimed for printing.

    

### 4.2. Application wide left menu usability
The left menu must be ACCESSIBLE from the SAME button on EVERY non-printable page of the application from the top bar. The left-menu must contain the most commonly used links in the application. The left menu must close itself when the user clicks outside of it.

    

#### 4.2.1. Left menu global configurability
The Admin of an instance must be able to fully configure the content of the left menu.

    

#### 4.2.2. Left menu personal customisability
Any user should be able to personally configure which are the left menu links, he/she would like to use. The personal links should be capable of fully overriding the links set by the administrator.

    

### 4.3. UI usability
The interaction of with the application UI must be as effortlessly, quickly and user-friendly as possible.

    

#### 4.3.1. Application wide top-bar usability
The top-bar of the application must be ALWAYS selected during a page load and particularly the omni-search box, so that the user could straight way start quick filtering the textual content of ANY page by typing at least 3 chars in the search box.
The top-bar must be visible on ALL pages, except those aimed for printing.

    

#### 4.3.2. Login page usability
The login page must contain clearly on which instance a user is logging in. The login must be executable both with enter and click of the GO button. The login page must load in less than 0.3s. The login after that to the home / landing page must not take more than 0.5 seconds. The error msgs must be as concise, but clear and explanatory as possible.

    

#### 4.3.3. Landing / home page usability
The landing / home page must clearly indicate that the user has logged in. It must contain clear UI element(s) to indicate where to go from here. The landing page might contain some additional informative content.

    

#### 4.3.4. View page usability
The view page must load in less than 0.3 s. 

    

### 4.4. Oneliner shell calls
The interaction of the application on the shell should be designed and implemented so that most of the features and bigger entry points should be accessible via one-liners on the shell - for example the testers should be able to lunch all the unit-tests via a single one line call. The integration tests should be triggerable via single oneline call. 

    

#### 4.4.1. Database recreation and DDL scripts run one-liners
The developers should be able to create the database via a single oneline call.

    

#### 4.4.2. Table(s) load via aa single one-liner
The developers should be able to load a table to the database via a single oneline call.

    

## 5. SCALABILITY


    

### 5.1. Horizontal performance scalability
The qto MUST support adding capacity horizontally in the application layer and by increasing database storage capacity simply by utilising the power of the cloud.

    

### 5.2. Feature scalability
The addition of new features should be as scalable as possible. The UI and control flow should be as generic as possible.

    

### 5.3. Setup scalability
The creation of new instances of the application should be as easy and automated as possible. 

    

### 5.4. Projects databases scalability
Each instance of the qto application must be able to connect to one or many project databases which DDL schemas matching the current api of the application.

    

### 5.5. Qto application clones scalability
The qto application must support cloning - that is "forking" into new applications with different names.

    

## 6. PERFORMANCE


    

### 6.1. Page load maximum time
Each page of the application containing less than 2000 items MUST load for less than 0.3 seconds.
Any new feature which does not meet this requirement should be disregarded or implemented into a clone of the application with different name ( see the cloning / forking section below ). 

    

### 6.2. Login, logout
Every login and logout operation MUST complete in less than 0.3 seconds in modern network environments. 

    

## 7. MULTI-INSTANCE OPERABILITY AND DEPLOYABILITY


    

### 7.1. Environment type self-awareness
Each deployed and running instance of the qto must "know" its own environment type - dev, tst, qas or prd to comply with the multi-instance architecture on a single host. 

    

### 7.2. Cross running between instances of different types
The application layers should support as much as possible cross running between different application layer instances and database instances - for example a dev application layer should be able to fetch data from a prd database.

    

## 8. UI REQUIREMENTS
The UI of the application must be fast, responsive, easy and pleasant to use.

    

### 8.1. CRUDs
The System must provide the needed UI interfaces to Create , Update , Delete and Search items in the system for the users having the privileges for those actions
Any modelled item in the database must be capable for:
 - create 
 - update
 - delete
 - search

    

#### 8.1.1. Execution time
The full execution time of any crud operation ( create, update, delete, search) from the end-user of the UI point of view should be less than 0.3 seconds

    

#### 8.1.2. Visual indication
The System should not show the so called ok messages, but only error messages ( with added verbosity on the console level ), yet the UI should be as responsive that the end-user would easily understand when an item has been created, updated or deleted.

    

### 8.2. Clarity on errors
The UI must present every error in a clear and concise way, so that the end-user would understand that an error has occurred, however no msgs should be displayed when the data is saved properly. 

    

### 8.3. Information search
The UI must enable easy to use and effective search for the end-users from all but the login page.

    

#### 8.3.1. Global project search
The UI must support global per project ( i.e. single project database ) text search from all but the login page.

    

#### 8.3.2. Quick search per page
Each information on each different type of search must be filterable from the omni search box - both the page content and the auxiliary left and/or right menu.

    

### 8.4. Login page requirements
All login error msgs should be clear and displayed with red colour. 

    

### 8.5. List page requirements
Each information on list page must be filterable from the omni search box - both the page content and the auxiliary left and/or right menu.

    

#### 8.5.1. View page requirements
Each information on view doc page must be filterable from the omni search box - both the page content and the auxiliary left and/or right menu.

    

## 9. INTEROPERABILITY
The qto application should support export and import from different data formats from both client and server side.

    

### 9.1. Data import


    

#### 9.1.1. Import from xls
The qto application should support a single xls data import.

    

#### 9.1.2. Import from json format
The qto application should support a single shell call json file to table import

    

#### 9.1.3. Files upload 
The qto application should provide an easy to use UI for uploading files, easily accessible from any page of the application. Once the file being uploaded the link to it should be provided to the user to be able to copy paste it into an issue , documentation etc object ...

    

### 9.2. Data export
The qto application must support export to different data formats via both the web interface or terminal access.

    

#### 9.2.1. Export to pdf
Any user having tcp access to an up-and-running qto instance must be able to export all or single item doc from that or another qto instance with tcp / ip connectivity into pdf files into a pre-configurable docs root directory both via ui or from an automation script.

    

#### 9.2.2. Export to md
Any user having tcp access to an up-and-running qto instance must be able to export all or single item doc from that or another qto instance with tcp / ip connectivity into md files into a pre-configurable docs root directory both via ui or from an automation script.

    

#### 9.2.3. Export to xls
Any user having tcp access to an up-and-running qto instance must be able to export all or single item doc from that or another qto instance with tcp / ip connectivity into Microsoft xls files into a pre-configurable docs root directory both via ui or from an automation script.

    

#### 9.2.4. Export to Microsoft docx
Any user having an up-and-running qto instance must be able to export all or single item doc from that or another qto instance with tcp / ip connectivity into Microsoft docx files into a pre-configurable docs root directory both via ui or from an automation script.

    

#### 9.2.5. Export to json
Any user having an up-and-running qto instance must be able to export all or single table from that or another qto instance with tcp/ip connectivity into json files.

    

## 10. SECURITY
A well operated instance of the qto application should have security corresponding to the data sensitivity it is operating on. 
For complete walkthrough of instance security check the following document: security_checklist_doc-0


    

### 10.1. Authentication
There should be the following 2 modes of authentication: 



    

#### 10.1.1. QTO_NO_AUTH mode
Any qto instance should support a non-authentication mode - that is all users having http and/or https access could perform all the actions on the UI without any restrictions, that is the customer organisation wanting own custom solution for authentication and authorisation should be able to run an instance with non-authorisation mode.
This mode MUST BE ALWAYS accessible to devops engineers to be able to quickly restart applications and troubleshoot them without having to add unnecessary complexity caused by the security aspects of the application.

    

#### 10.1.2. Simple Native authentication mode

All registered users should have access to all but users related data. 

If a user is not registered the error msg to the login should prompt him which e-mail to contact to be registered ( which will be the e-mail of the product owner instance ). 

If the admin user is able to impersonate another user it must simply mean that he/she has done that on purpose ( aka maliciously )
The sessions of different dev, tst and prod app layer instances should not intermix within the multiple open processes / threads of the same browser of the same user.


    

##### 10.1.2.1. User email and password matching for login success

Users should login with email and password. Users' names and other personal data MUST NOT be tracked by the application. 
Unregistered users should have access to the login page only. 

    

##### 10.1.2.2. The passwords must NOT be stored in clear text
The application must match the passwords via blowfish encryption and store the authentication details into the session of default of 10h.

    

##### 10.1.2.3. name ...


    

##### 10.1.2.4. Passwords sensibility

The regular users should see only their credentials. Only the admin user should see all the users credentials , but with the passwords encrypted.

    

#### 10.1.3. RBAC authentication mode via JWT
The qto application should support native web tokens based authentication, by using as login a valid user e-mail and password, stored in the qto instance database. 

The qto should support SSO authentication as described in the following RFC's. 
The Users should be authenticated by means of the most simplest OAauth2.0 authentication flow: 
https://tools.ietf.org/html/rfc6749#section-5.1

    

##### 10.1.3.1. Login
The login must be done against email and password over https. Should the email and the pass not match, a 401 http code ( UnAuthorised ) must be returned.

    

##### 10.1.3.2. Must be stateless and thus horizontally scalable
The JSON web token authentication must be stateless to enable both process ( aka run-time ) and hosts scalability. The JWT must be signed with private key against tampering.
Token verification must be done without db lookup and only in-memory. 

    

##### 10.1.3.3. Must not use permanent cookies
Because permanent cookies might be hacked by js scripts which are running in the same browser instance ...

Must use HttpOnly cookies: 
https://developer.mozilla.org/en-US/docs/Web/HTTP/Cookies

    

##### 10.1.3.4. Must not use localStore, but Authorisation header
But rather some combination of httpOnly session and JWT , same-origin
Than CORS will not be an issue - https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS , must stay operational even if.

    

##### 10.1.3.5. Must have arymetric signature
The JSON web tokens must be viewable from the other parties by . Must support public and private key decoding

    

##### 10.1.3.6. The tokens must be self-contained
The tokens must be self-contained, i.e. the token must carry all the user data needed for the authorisation in is payload, but MUST not expose the site to XSS.
The tokens must be set in the Authorisation header.

    

##### 10.1.3.7. Tokens expiration
The tokens could request refresh tokens once the expiration time is close.

    

##### 10.1.3.8. Must support whitelisting
Whitelisting is the practice of explicitly allowing some identified entities access to a particular privilege, service, mobility, access or recognition.
A qto admin must be able to revoke access to user token or by username or by token id.

    

##### 10.1.3.9. Must support black-listing
Blacklisting is the action of a group or authority, compiling a blacklist (or black list) of people, countries or other entities to be avoided or distrusted as not being acceptable to those making the list.[1] A blacklist can list people to be discriminated against, refused employment, or censored.
A qto admin must be able to revoke access from specific users and specific user tokens.

    

### 10.2. Authorisation
The Qto application should have authorisation as described in the RFC 6749.

    

#### 10.2.1. QTO_NO_AUTH mode authorisation
There must be no authorisation in the non-authentication mode - everyone must be  allowed to do anything, including truncating tables etc.

    

#### 10.2.2. Simple Native mode authorisation
There must be almost no authorisation in the Simple Native mode authorisation mode - everyone must be  allowed to do anything, including truncating tables etc. except actions on the users table.

    

#### 10.2.3. RBAC mode authorisation


    

##### 10.2.3.1. Role-based Access Control
The Access Control in the qto application in the RBAC mode authorisation mode must be role-based. The application must authorise the usage of resources based on fully configurable per role , per resource , per route.

    

##### 10.2.3.2. Authorisation via Json Web Tokens during login
The application must create JWT during login which will must be signed with a private key and will contain the minimum set of claims to support the Roles-Based Access Control.

    

##### 10.2.3.3. Roles based Access Control list for routes and items
The application must provide the means for project owners to define who has access to what resources and why via a centralised UI.

    

##### 10.2.3.4. Role Based Access Control on tables row level
The application must provide a revocable traditional permissions model for tables on table rows level. The "non-enforceable" means that no overhead work must be enforced on organisations not-using the model at all. 

    

##### 10.2.3.5. Support for segregation of duties
The application must support segregation of duties during deployment, that is the team which delivers the new software version / configuration change must be able to trigger the deployment to production without actually having access to the production data.

    

#### 10.2.4. Role level based security 
The qto must support row level based security : 
https://www.postgresql.org/docs/current/ddl-rowsecurity.html

    

## 11. DOCUMENTATION


    

### 11.1. Documentation completeness
Each running instance MUST have the following documentation set :
 - End User Guide 
- Installation and Configuration Guide doc
- DevOps Guide doc
- Requirements doc
- System Guide doc
- UserStories doc
- Maintenance and Operations Guide doc

in the following formats: 
 - native qto view-doc page format ( as soon as the instance is up-and-running )
 - md format - as soon as the source code is downloaded 


    

### 11.2. Documentation and code base synchronization
Each running instance MUST have its required documentation set up-to-date for it's release version. No undocumented or hidden features are allowed. Should any be missing or misreported a new issue must be created to correct those with top priority. 

    

#### 11.2.1. Requirements push
Whenever a project database meta-data is updated a new "do reload the current page" should be pushed on all the clients having currently session in the application …

    

<file_sep># src/bash/qto/funcs/check-perl-syntax.test.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doTestCheckPerlSyntax comments ...
# ---------------------------------------------------------
doTestCheckPerlSyntax(){

	do_log "DEBUG START doTestCheckPerlSyntax"
	
	cat doc/txt/qto/tests/perl/check-perl-syntax.test.txt
	sleep "$sleep_interval"

	bash src/bash/qto/qto.sh -a check-perl-syntax
   exit_code=$?
	do_log " check-perl-syntax-1 exit_code: $exit_code "

   sleep "$sleep_interval"
   test $exit_code -ne 0 && return

	do_log "DEBUG STOP  doTestCheckPerlSyntax"
}
# eof func doTestCheckPerlSyntax


# eof file: src/bash/qto/funcs/check-perl-syntax.test.sh
<file_sep>#!/usr/bin/env bash

clear ; ./src/bash/qto/qto.sh -a unscramble-confs
<file_sep># src/bash/qto/funcs/backup-postgres-db.test.sh

doTestBackupPostgresDb(){

	do_log "DEBUG START doTestBackupPostgresDb"
	
	# Action !!!
   bash src/bash/qto/qto.sh -a backup-postgres-db
	sleep "$sleep_interval"
   
   # there should be at least one file in the current daily dir which is not older than 
   while read -r f; do 
      count=$(( count + 1 ))
   done < <(find $mix_data_dir/`date "+%Y"`/`date "+%Y-%m"`/`date "+%Y-%m-%d"`/sql/$postgres_app_db -name '*full.dmp.sql' -mmin -1)

   if [ $count -lt 1 ]; then
      msg="db dump files are not found !!!"
      export exit_code=1 ;
      do_exit "$msg";
      exit 1 ;
   fi
   
   msg="at least one db dump file was created"
   test $exit_code -ne 0 && return
 
	do_log "DEBUG STOP  doTestBackupPostgresDb"
}
# eof func doTestBackupPostgresDb


# eof file: src/bash/qto/funcs/backup-postgres-db.test.sh
<file_sep>.PHONY: do-prune-docker-system ## @-> stop & completely wipe out all the docker caches for ALL IMAGES !!!
do-prune-docker-system:
	@clear
	-docker kill $$(docker ps -q)
	-docker rm $$(docker ps -aq)
	docker image prune -a -f
	docker builder prune -f -a
	docker system prune --volumes -f
<file_sep># src/bash/qto/funcs/change-version.help.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doHelpChangeVersion comments ...
# ---------------------------------------------------------
doHelpChangeVersion(){

	do_log "DEBUG START doHelpChangeVersion"
	
	cat doc/txt/qto/helps/change-version.help.txt
	sleep 2
	# add your action implementation code here ... 

	do_log "DEBUG STOP  doHelpChangeVersion"
}
# eof func doHelpChangeVersion


# eof file: src/bash/qto/funcs/change-version.help.sh
<file_sep>#!/usr/bin/env bash

#------------------------------------------------------------------------------
# increase or decrease the version of the product
# clone the <<base_dir>>/app_name/a
# bash src/bash/qto/qto.sh -a to-app=<<new_app>>
#------------------------------------------------------------------------------
doCloneToApp(){
	tgt_app="$1"
	prefix='to-app='
	tgt_app=${tgt_app#$prefix}
	tgt_environment_name=$(echo $environment_name | perl -ne "s/$RUN_UNIT/$tgt_app/g;print")
	tgt_environment_name=$(echo $tgt_environment_name | perl -ne "s/$product_version/0.0.0/g;print")
	tgt_environment_name=$(echo $tgt_environment_name | perl -ne "s/$ENV/dev/g;print")
	tgt_product_dir=$product_base_dir/$tgt_app
	tgt_PRODUCT_DIR=$tgt_product_dir/$tgt_environment_name

	mkdir -p $tgt_PRODUCT_DIR 
	[[ $? -eq 0 ]] || do_exit 2 "ERROR :: cannot create the tgt_PRODUCT_DIR: $tgt_PRODUCT_DIR !"

	# remove everything from the tgt product version dir - no extra files allowed !!!
	rm -fvr $tgt_PRODUCT_DIR
	# if the removal failed exit with error msg
	[[ $? -eq 0 ]] || do_exit 2 "ERROR :: cannot write to $tgt_PRODUCT_DIR !"

	doCreateRelativePackage
	unzip -o $zip_file -d $tgt_PRODUCT_DIR

	to_srch=$RUN_UNIT
	to_repl=$tgt_app

	#-- search and replace in file names
	find "$tgt_PRODUCT_DIR/" -type d |\
		perl -nle '$o=$_;s#'"$to_srch"'#'"$to_repl"'#g;$n=$_;`mkdir -p $n` ;'
	find "$tgt_PRODUCT_DIR/" -type f |\
		perl -nle '$o=$_;s#'"$to_srch"'#'"$to_repl"'#g;$n=$_;rename($o,$n) unless -e $n ;'


	find $tgt_PRODUCT_DIR -exec file {} \; | grep text | cut -d: -f1| { while read -r file_to_edit ;
		do (
			perl -pi -e "s#$to_srch#$to_repl#g" "$file_to_edit"
		);
		done ;
	}
	
	# on cygwin the perl -pi leaves backup files => remove them
	find $tgt_PRODUCT_DIR -type f -name '*.bak' | xargs rm -fv 

	cp -v $zip_file $tgt_PRODUCT_DIR

}
<file_sep>-- \echo 'If necessary, perform -- DROP TABLE IF EXISTS app_user_roles;'

-- \echo '2. Creating the app_user_roles table'

CREATE TABLE app_user_roles (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , app_users_guid     UUID NOT NULL DEFAULT '2660a6e9-9e6b-4faa-8264-27a92872657b'
    , app_roles_guid     UUID NOT NULL DEFAULT '71eea083-d818-4557-89fe-29eb950881ab'
    , name           varchar (100) NOT NULL DEFAULT 'Role name'
    , description    varchar (200) NULL DEFAULT ''
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_app_user_roles_guid PRIMARY KEY (guid)
    );

CREATE UNIQUE INDEX idx_app_user_roles_uniq_id
	ON app_user_roles (id);


-- \echo 'If necessary, perform ALTER TABLE public.app_user_roles -- DROP CONSTRAINT fk_app_users_guid;'

ALTER TABLE public.app_user_roles
   ADD CONSTRAINT fk_app_users_guid 
   FOREIGN KEY (app_users_guid)
   REFERENCES app_users(guid) 
   ON DELETE CASCADE;
;
	
ALTER TABLE public.app_user_roles
   ADD CONSTRAINT fk_app_roles_guid 
   FOREIGN KEY (app_roles_guid)
   REFERENCES app_roles(guid) 
   ON DELETE CASCADE;
;

-- \echo 'List columns of the created table app_user_roles');

SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid='public.app_user_roles'::regclass
   AND    attnum>0
   ORDER  BY attnum; 

-- \echo 'Update time on every EXECUTE trigger:'
CREATE TRIGGER trg_set_update_time_on_app_user_roles
	BEFORE UPDATE ON app_user_roles 
	FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
	FROM pg_trigger
	WHERE NOT tgisinternal
	AND tgrelid='app_user_roles'::regclass;

--
-- TOC entry 3355 (class 0 OID 137804)
-- Dependencies: 282
-- Data for Name: app_user_roles; Type: TABLE DATA; Schema: public; Owner: usrdevqtoadmin
--

COPY public.app_user_roles (guid, id, app_users_guid, app_roles_guid, name, description, update_time) FROM stdin;
6f036963-52cc-4d49-ae36-984c75d14bb2	200411160810	2660a6e9-9e6b-4faa-8264-27a92872657c	71eea083-d818-4557-89fe-29eb950881ad	trd must be reader	Test reader user has reader role	2020-05-01 21:13:48
0fb84503-0d7a-4cdc-889b-4e4a8f6aac53	200410220814	2660a6e9-9e6b-4faa-8264-27a92872657d	71eea083-d818-4557-89fe-29eb950881ac	ted must be editor	Test editor user has editor role	2020-05-01 21:14:01
460cc595-7db7-4211-a271-d825b3801d1e	200224195828	2660a6e9-9e6b-4faa-8264-27a92872657b	71eea083-d818-4557-89fe-29eb950881ab	tau must be anonymous	Test anonymous user has anonymous role	2020-05-01 21:14:11
472cbd25-d8a5-4828-85f3-556387aea673	200502001551	9426c566-0abd-4028-830f-82ed9f91d262	71eea083-d818-4557-89fe-29eb950881aa	pio must be product instance owner	Test product instance owner has instance owner role	2020-05-01 21:19:21
\.
<file_sep>#v1.0.7 
#------------------------------------------------------------------------------
# cleans the unneeded during after run-time stuff
#------------------------------------------------------------------------------
doPrintHelp(){

   doSetVars

   do_flush_screen

   test -z $target_dir && target_dir='<<target_dir>>'

   cat <<END_HELP

   #------------------------------------------------------------------------------
   ## START HELP `basename $0`
   #------------------------------------------------------------------------------
      `basename $0` is a the minimalistic scala console stub app to start writing scala
      `basename $0` is is also an utility script with the goodies listed below:

      # go get this help, albeit you knew that already ...
      bash $0 -h
      or
      bash $0 --help

		5. this will create the deployment package into your production version dir: $product_dir
		if you have configured the network_backup_dir in cnf file it will be also copied 
      #--------------------------------------------------------
		bash $0 -a create-deployment-package

		You must specify the files to be be included in the full package from the met/qto.dev 
		file
      
		6. to create a relative package
		this will create the relative package into your production version dir: $product_dir
		if you have configured the network_backup_dir in cnf file it will be also copied 
      #--------------------------------------------------------
		bash $0 -a create-relative-package

		You must specify the files to be be included in the full package from the met/qto.dev 
		file
      
		4. to clone the same version of your product into dev,tst,prd enviroments
      #--------------------------------------------------------
      bash $0 -a to-dev
      bash $0 -a to-tst
      bash $0 -a to-qas
      bash $0 -a to-prd
      
		5. to clone the product into a different version
      #--------------------------------------------------------
      bash $0 -a to-ver=0.1.1
      
		6. to create the tags file in the \$product_dir : $product_dir
      #--------------------------------------------------------
      sh $0 -a create-ctags

		
		9. to save your current tmux session run the following command
      #--------------------------------------------------------
      sh $0 -a save-tmux-session

      #------------------------------------------------------------------------------
      ## INSTALLATION
      #------------------------------------------------------------------------------

		Installation is as simple as unzip -o <<full_package>> -d <<desired_base_dir>>
		where <<full_package>> should look like: qto.0.0.2.prd.20160702_181412.ysg-host-name.zip
		and <<disired_base_dir>> could be /tmp , /opt/ , /var , /even/longer/path
		The required binaries for the $RUN_UNIT are:
		- perl
		If you want to use the vim's jump to tag function you would have to install also:
		- ctags
		

      #------------------------------------------------------------------------------
      ## FOR DEVELOPERS
      #------------------------------------------------------------------------------

      `basename $0` is an utility script having the following purpose
      to provide an easy installable starting template for any app
		based on the following philosophy:
		- there is a huge difference between a software product and a running instance
		- the running istance is properly configured it usually belongs to dev , tst 
		  prd environments 
		- different version of the same software product might or might not require 
		  different binaries 
		- if two instances of the same product having different versions can operate on 
		  the same host simultaniously 
		- in $RUN_UNIT this ability to run multiple versions in multiple environments
        under even different *Nix users is in-built
		- any custom software should be installed and run from a base dir: $base_dir

      with  the following functionalities:
      - printing help with cmd switch -h ( verify with doTestHelp in test-sh )
      - prints the set in the script variables set during run-time
      - separation of host specific vars into separate configuration file :
       <<wrap_bash_dir>>/<<RUN_UNIT>>.<<MyHost>>.cnf
       $ini_file
      - thus easier enabling portability between hosts
      - logging on terminal and into configurable log file set now as:
       $log_file
      - for loop examples with head removal and inline find and replace
      - cmd args parsing
      - doSendReport func to the tail from the log file to pre-configured emails
      - support for parallel run by multiple processes - each process uses its own tmp dir

	Note the help is quite long - you might want to use the less page :
      `basename $0` --help \| less
   #------------------------------------------------------------------------------
   ## STOP HELP `basename $0`
   #------------------------------------------------------------------------------

END_HELP
}
#eof doPrintHelp 
<file_sep>-- start src: http://rachbelaid.com/postgres-full-text-search-is-good-enough/
-- build the document
/*
SELECT to_tsvector(name) || to_tsvector(description) as document 
FROM monthly_issues 
; 
*/

-- search 
SELECT id, 'monthly_issues' as table_name , name , ts_rank(to_tsvector(name || description), to_tsquery('search')) as relevancy
FROM monthly_issues 
WHERE 1=1 
AND ts_rank(to_tsvector(name || description), to_tsquery('search')) <> 0
UNION
SELECT id, 'yearly_issues' as table_name , name , ts_rank(to_tsvector(name || description), to_tsquery('search')) as relevancy
FROM yearly_issues 
WHERE 1=1 
AND ts_rank(to_tsvector(name || description), to_tsquery('search')) <> 0
ORDER BY relevancy desc
; 

-- stop  src: http://rachbelaid.com/postgres-full-text-search-is-good-enough/



/*
SELECT setweight(to_tsvector('english', COALESCE(name,'')), 'A') ||  
     setweight(to_tsvector('english', COALESCE(description,'')), 'A')
FROM monthly_issues 
;
DROP MATERIALIZED VIEW mvw_monthly_issues
;
CREATE MATERIALIZED VIEW mvw_monthly_issues
AS
SELECT id, to_tsvector(concat_ws(' ', name, description)) AS tsv_monthly_issues
FROM monthly_issues
;
SELECT * FROM mvw_monthly_issues WHERE tsv_monthly_issues @@ plainto_tsquery('search')
;
*/
/*
SELECT to_tsvector('english', name || description)  
FROM monthly_issues 
*/
; 
/*
WITH
    q AS ( SELECT plainto_tsquery('English', 'create') AS query ),
    d AS ( SELECT (name || ' ' || description) AS document FROM monthly_issues ),
    t AS ( SELECT to_tsvector('English', d.document) AS textsearch FROM d ),
    r AS ( SELECT ts_rank_cd(t.textsearch, q.query) AS rank FROM t, q )
SELECT monthly_issues.name , ts_headline('English', d.document, q.query) AS matches
FROM monthly_issues, q, d, t , r
WHERE q.query @@ t.textsearch 
ORDER BY r.rank DESC 
LIMIT 5;
*/
<file_sep>-- DROP TABLE IF EXISTS meta_cols ; 
-- OBS - this is NOT THE app_item_columns table 
-- use this one to populate meta data from different dbs ... just like data ... 

SELECT 'create the "meta_cols" table'
; 
   CREATE TABLE meta_cols (
      guid              UUID NOT NULL DEFAULT gen_random_uuid()
    , id                bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , table_name        varchar (100) NOT NULL DEFAULT 'src_table_name...'
    , prio              integer NOT NULL DEFAULT 1
    , ordinal_number    integer NOT NULL DEFAULT 1
    , src_table_name        varchar (100) NOT NULL DEFAULT 'src_table_name...'
    , tgt_table_name        varchar (100) NOT NULL DEFAULT 'tgt_table_name...'
    , name              varchar (100) NOT NULL DEFAULT 'col-name...'
    , json_file         varchar (100) NOT NULL DEFAULT 'json_file...'
    , json_expression   varchar (100) NOT NULL DEFAULT 'json_expression...'
    , description       varchar (4000)
    , update_time       timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_meta_cols_guid PRIMARY KEY (guid)
    , CONSTRAINT uc_table_column unique (table_name, name)
    ) WITH (
      OIDS=FALSE
    );

create unique index idx_uniq_meta_cols_id on meta_cols (id);



SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.meta_cols'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 

CREATE TRIGGER trg_set_update_time_on_meta_cols 
   BEFORE UPDATE ON meta_cols FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'meta_cols'::regclass;

<file_sep>// Define a new component called button-counter
Vue.component('btn-new', {
  data: function () {
    return {
      count: 0
    }
  },
  methods: {
      clickedBtnNew() {
         function pad2(n) { return n < 10 ? '0' + n : n }
         var date = new Date();
         var dbid = date.getFullYear().toString().substr(2,3) + pad2(date.getMonth() + 1) + pad2( date.getDate()) + pad2( date.getHours() ) + pad2( date.getMinutes() ) + pad2( date.getSeconds() );
         console.log ( "clicked with id: " + dbid)
			this.$emit('clickedBtnNew',dbid)

      }
  },
  template: `<button v-on:click="clickedBtnNew()">create new</button>`
})

// register the editable
Vue.component('editable', {
  template: `
    <div v-bind:id="id" contenteditable="true" @blur="emitChange">
      {{ content }}
    </div>
  `,
  props: ['content','id'],
  methods: {
    emitChange (ev) {
      this.$emit('update', ev.target.textContent,ev.target.id)
    }
  }
})
// register the grid component
Vue.component('demo-grid', {
  template: '#grid-template',
  props: {
    data: Array,
    columns: Array,
    filterKey: String
  },
  data: function () {
    var sortOrders = {}
    this.columns.forEach(function (key) {
      sortOrders[key] = 1
    })
    return {
      sortKey: '',
      sortOrders: sortOrders
    }
  },
  computed: {
    filteredData: function () {
      var sortKey = this.sortKey
      var filterKey = this.filterKey && this.filterKey.toLowerCase()
      var order = this.sortOrders[sortKey] || 1
      var data = this.data
      if (filterKey) {
        data = data.filter(function (row) {
          return Object.keys(row).some(function (key) {
            return String(row[key]).toLowerCase().indexOf(filterKey) > -1
          })
        })
      }
      if (sortKey) {
        data = data.slice().sort(function (a, b) {
          a = a[sortKey]
          b = b[sortKey]
          return (a === b ? 0 : a > b ? 1 : -1) * order
        })
      }
      return data
    }
  },
  filters: {
    capitalize: function (str) {
      return str.charAt(0).toUpperCase() + str.slice(1)
    }
  },
  methods: {
    sortBy: function (key) {
      this.sortKey = key
      this.sortOrders[key] = this.sortOrders[key] * -1
    },
    updateItem (content,id) {
      var arr = id.split('-')
      var col = arr[0]
      var dbid = arr[1]
		console.log ( "new content: " + content ) 
		console.log ( "column: " + arr[0] )
		console.log ( "dbid: " + arr[1] )
      console.log ( "should run: UPDATE monthly_issues set " + col + " = '" + content + "' WHERE id='" + dbid + "'" )
      console.log ( "/dev_qto/update/monthly_issues?pick=id" + arr[1] ) 


         axios.post('/dev_qto/update/monthly_issues', {
             attribute: col,
             id: dbid
         })
         .then(function (response) {
            console.log(response);
         })
         .catch(function (error) {
            console.log(error);
          });

    },
    createNewItem (dbid) {
		axios.post('/dev_qto/create/monthly_issues', {
			 id: dbid
		})
		.then(function (response) {
			console.log(response);
		})
		.catch(function (error) {
			console.log(error);
		 });

    }
  }
})

// bootstrap the demo
var demo = new Vue({
  el: '#demo',
  data: {
    searchQuery: '',
    gridColumns: ['id', 'name', 'prio','description'],
    gridData: []
  },
  mounted() {
    axios.get("/dev_qto/select/monthly_issues?pick=id,name,prio,description")
    .then(response => {this.gridData = response.data.dat
	 })
 },
})
<file_sep># file: src/bash/qto/funcs/remove-all-docker-containers.func.sh

# v0.6.5
# ---------------------------------------------------------
# remove all the docker containers on the host with warning
# and inform if none exists
# ---------------------------------------------------------
doRemoveDockerContainers(){

   do_flush_screen
   do_log "INFO START removing all docker containers with the $product_version.$ENV tag !!! " 
   do_log "INFO Are you SURE ??!! You have 3 seconds to abort by Ctrl + C !!"
   sleep 3

   ret=$(docker ps -a|grep -i qto-image.$ENV|awk '{print $1}'|wc -l| perl -ne 's/\s+//g;print')
   if test $ret -eq 0 
   then
      do_log "INFO No docker containers found. Nothing to do !!!"
   else
      do_log "INFO stopping containers ..."
      docker stop $(docker ps -a|grep -i qto-image:$product_version.$ENV|awk '{print $1}')
      sleep 1
      do_log "INFO removing containers ..."
      docker rm $(docker ps -a|grep -i qto-image:$product_version.$ENV|awk '{print $1}')
   fi

	do_log "INFO STOP  removing all docker containers"
}
<file_sep>#!/bin/bash

do_create_ssh_keys(){


   test -z ${PROJ_CONF_FILE:-} && export PROJ_CONF_FILE="$PRODUCT_DIR/cnf/env/$ENV.env.json"
   source $PRODUCT_DIR/lib/bash/funcs/export-json-section-vars.sh
   do_export_json_section_vars $PROJ_CONF_FILE '.env.aws'

   which expect || sudo apt-get update && sudo apt-get install -y expect


   for ENV in `echo dev tst prd`; do
      jwt_private_key_file=~/.ssh/qto.$ENV.jwtRS256.key
      jwt_public_key_file=~/.ssh/qto.$ENV.jwtRS256.key.pub
      echo << EOF_USING
      creating and using the following private and public key files for the Guardian's JWT's :
      $jwt_private_key_file
      $jwt_public_key_file
EOF_USING

      test -f $jwt_private_key_file || ssh-keygen -t rsa -b 4096 -m PEM -f $jwt_private_key_file -N ''
      test -f $jwt_public_key_file && rm -v $jwt_public_key_file
      openssl rsa -in $jwt_private_key_file -pubout -outform PEM -out $jwt_public_key_file
   done;



   private_ssh_key_fpath_no_env=$(echo $private_ssh_key_fpath|rev|cut -c 5-|rev)
   private_ssh_key_fpath_no_env=$(eval echo $private_ssh_key_fpath_no_env)


   for ENV in `echo dev tst prd`; do
      test -f $private_ssh_key_fpath_no_env'.'$ENV || {
         ssh-keygen -q -t rsa -b 4096 -N '' -f $private_ssh_key_fpath_no_env.$ENV <<< ""$'\n'"y" 2>&1 >/dev/null
      }
   done

}
<file_sep># src/bash/qto/funcs/run-pgsql-scripts.test.sh

doTestRunPgsqlScripts(){

	do_log "DEBUG START doTestRunPgsqlScripts"
	
	test -z "$sleep_interval" || sleep "$sleep_interval"

	do_log "test the running of the sql scripts under the product instance dir"
	do_log "Action !!! "
	do_log "bash src/bash/qto/qto.sh -a run-pgsql-scripts"

	bash src/bash/qto/qto.sh -a run-pgsql-scripts
   exit_code=$?
   test $exit_code -ne 0 && do_exit $exit_code $exit_msg
   
	do_log "test the running of the sql scripts outside the product instance dir"
	do_log "with pre-set exported pgsql_scripts_dir var : aka export pgsql_scripts_dir=/tmp/qto"

   mkdir -p /tmp/qto
   cp -vr src/sql/pgsql/dev_pgsql_runner/* /tmp/qto/   
   export pgsql_scripts_dir=/tmp/qto

	do_log "Action !!! "
	do_log "bash src/bash/qto/qto.sh -a run-pgsql-scripts"
	bash src/bash/qto/qto.sh -a run-pgsql-scripts
   exit_code=$?
   do_log "DEBUG STOP  doTestRunPgsqlScripts"
}


# eof file: src/bash/qto/funcs/run-pgsql-scripts.test.sh
<file_sep>doSearchAndReplaceTokens(){

   test -d $dir || dir=$PRODUCT_DIR/src/sql/

   while read -r l ; do 
      to_srch=$(echo $l|awk '{print $1}')
      to_repl=$(echo $l|awk '{print $2}')

      while read -r f ; do 
         perl -pi -e "s/$to_srch/$to_repl/gm" $f
      done < <(find $dir -type f| grep -v search-and-replace-tokens.func.sh)

   # space or tabs do not mather - to search on the left, to replace on the right
   done < <(cat << EOF_WORDS
         meta_tables                app_items
         meta_columns               app_item_attributes
         meta_routes                app_routes
         roles                      app_roles
         user_roles                 app_user_roles
         items_roles_permissions    app_items_roles_permissions
         items_app_roles_permissions app_items_roles_permissions
         pg_app_roles               pg_roles
         app_app_                   app_
         user_app_roles             app_user_roles
EOF_WORDS
)

}
#         items                      app_items
#         item_attributes            app_item_attributes
#         routes                     app_routes
#         roles                      app_roles
#         user_roles                 app_user_roles
#         items_roles_permissions    app_items_roles_permissions
#         items_app_roles_permissions app_items_roles_permissions
<file_sep>#  QTO SYSTEM GUIDE
* [1. INTRO](#1-intro)
  * [1.1. PURPOSE](#11-purpose)
  * [1.2. AUDIENCE](#12-audience)
* [2. THE INFORMATION SYSTEM AS CYBERNETIC ORGANISM ABSTRACTION](#2-the-information-system-as-cybernetic-organism-abstraction)
  * [2.1. THE QTO INFORMATION SYSTEM - DEFINITION](#21-the-qto-information-system--definition)
  * [2.2. INFORMATION SYSTEM - DIAGRAM](#22-information-system--diagram)
* [3. THE QTO SYSTEM INFRASTRUCTURE](#3-the-qto-system-infrastructure)
  * [3.1. SYSTEM ARCHITECTURAL OVERVIEW BY SIWA DIA](#31-system-architectural-overview-by-siwa-dia)
  * [3.2. INFRASTRUCTURAL COMPONENTS](#32-infrastructural-components)
    * [3.2.1. End-user clients](#321-end-user-clients)
    * [3.2.2. Application Layer and databases hosted in AWS](#322-application-layer-and-databases-hosted-in-aws)
    * [3.2.3. Local Development and testing](#323-local-development-and-testing)
    * [3.2.4. Source code in GitHub](#324-source-code-in-github)
* [4. ARCHITECTURE](#4-architecture)
  * [4.1. ICOCM ARCHITECTURE DEFINITION](#41-icocm-architecture-definition)
    * [4.1.1. The Control components](#411-the-control-components)
    * [4.1.2. The Model components](#412-the-model-components)
    * [4.1.3. The Input Components](#413-the-input-components)
    * [4.1.4. The Output Components](#414-the-output-components)
    * [4.1.5. The Conversion Components](#415-the-conversion-components)
    * [4.1.6. The Converter Components](#416-the-converter-components)
  * [4.2. MULTI-INSTANCE SETUP](#42-multi-instance-setup)
    * [4.2.1. Multi-environment naming convention](#421-multi-environment-naming-convention)
  * [4.3. SOFWARE ARCHITECTURE](#43-sofware-architecture)
    * [4.3.1. Front-End](#431-front-end)
    * [4.3.2. Back-End](#432-back-end)
  * [4.4. HOT META DATA RELOAD FUNCTIONALITY](#44-hot-meta-data-reload-functionality)
* [5. FULL STACK DESCRIPTION](#5-full-stack-description)
  * [5.1. OS STACK](#51-os-stack)
  * [5.2. THE APPLICATION LAYER STACK](#52-the-application-layer-stack)
    * [5.2.1. Perl](#521-perl)
    * [5.2.2. The perl modules](#522-the-perl-modules)
  * [5.3. THE MOJOLICIOUS WEB FRAMEWORK](#53-the-mojolicious-web-framework)
  * [5.4. THE BROWSER RUN-TIME](#54-the-browser-run-time)
    * [5.4.1. HTML 5](#541-html-5)
    * [5.4.2. JavaScript ](#542-javascript-)
      * [5.4.2.1. Vue](#5421-vue)
* [6. APPLICATION CONTROL FLOW ](#6-application-control-flow-)
  * [6.1. SHELL CONTROL FLOW](#61-shell-control-flow)
    * [6.1.1. Front-End](#611-front-end)
    * [6.1.2. Back-End](#612-back-end)
* [7. SECURITY](#7-security)
  * [7.1. NON-SECURITY MODE](#71-non-security-mode)
  * [7.2. SIMPLE NATIVE SECURITY MODE](#72-simple-native-security-mode)




    

## 1. INTRO


    

### 1.1. Purpose
The purpose of this guide is to provide description of the qto system and application architecture.

    

### 1.2. Audience
Target audience of this document is comprised of the architects and System designers of a potential or current system, comprised on deployed and operating qto instances. Developers and devops operators.

    

## 2. THE INFORMATION SYSTEM AS CYBERNETIC ORGANISM ABSTRACTION
This section is essential for you to understand the very basic motives and principles behind the design of the qto applications and systems. <NAME> stated in several interviews, that any organisation might be viewed as a cybernetic organism, ran to a certain degree by an Artificial Intelligence Instance... 
Qto has nothing to do with the modern AI definition, yet it's design is inspired by this cybernetic organism or biome metaphor. 

    

### 2.1. The qto Information System - definition
There are multiple definitions for "Information System" - most of those apply to the qto information system, yet the following definition provides additional content, which might not necessary be applicable to any other non-qto IS. 
The qto information system is the interactive combination of the people , processes & policies, hardware & networks , source code, binaries, configuration and data managed by any running instance of the qto application.

    

### 2.2. Information System - diagram
The following diagram depicts the mention above definition.

    

## 3. THE QTO SYSTEM INFRASTRUCTURE
This section describes the current system infrastructure.

    

### 3.1. System architectural overview by SIWA dia
The following diagram implements the Simplest Possible Way of describing Architecture principle - it's sole purpose is to quickly provide an overview of the existing infrastructure built with the help of the qto application as well as provide visual tool for communication related to thejyot application. 

    

### 3.2. Infrastructural components


    

#### 3.2.1. End-user clients
The end-users can access any qto application via their browsers. All of the functionalities are available in mobile browsers ( smart-phones or tablets not older than 2 years)

    

#### 3.2.2. Application Layer and databases hosted in AWS
Both the application layer and the database(s) are hosted on the same amazon ec2 host ( this will change in the future, as the architecture supports databases hosting in RDS or in separate hosts with TCP data channel)

    

#### 3.2.3. Local Development and testing
The development and testing are done in an ubuntu 18.04 vm running on top of mac - the binary configuration for the vm in described in the bootstrap script.
You could also develop and test in other Unix-like OS('s) - GentOs, MacOS etc, as long as you could figure out how-to install and provision postgres , the required OS binaries and the Perl modules for the application layer, as the provided deployment script has been aimed and tested only for the latest Ubuntu LTS.

    

#### 3.2.4. Source code in GitHub
The source code for the qto project is hosted in GitHub with the most open licensing possible:
https://github.com/YordanGeorgiev/qto



    

## 4. ARCHITECTURE


    

### 4.1. ICOCM architecture definition
The Input-Output Control Model architecture is and application architecture providing the highest possible abstraction for almost any software artefact, by dividing its main executing components based on their abstract responsibilities - Input, Conversion, Output , Control and Model. 

    

#### 4.1.1. The Control components
The Control components control the control flow in the application. The instantiate the Models and pass them to the Readers , Converters and Writers for output. 

    

#### 4.1.2. The Model components
The model components model the global DATA of the application - that is no configuration, nor control flow nor anything else should be contained within the model. 
Should you encounter data, which is not modelled yet , you should expand the Model and NOT provide different data storage and passing techniques elsewhere in the code base ... 

    

#### 4.1.3. The Input Components
The Input Components are generally named as "Readers". Their responsibility is to read the application data into Model(s). 

    

#### 4.1.4. The Output Components
The Output Components are generally named as "Writers" Their responsibility is to write the already processed data from the Models into the output media . 

    

#### 4.1.5. The Conversion Components
The Conversion components are generally called "converters". Their responsibility is to convert from one run-time data structure to another.

    

#### 4.1.6. The Converter Components
The Converters apply usually the business logic for converting the input data from the Models into the app specific data back to the Models. 

    

### 4.2. Multi-instance setup
The multi-instance setup refers to the capability of any installed and setup instance of the qto application to "know" its version , environment type  - development , testing and production ) and owner.

    

#### 4.2.1. Multi-environment naming convention
Each database used by the qto application has an &lt;&lt;environment abbreviation&gt;&gt; suffix referring to its environment type. Running application layers against different db versions should be supported as much as possible.  

    

### 4.3. Sofware architecture


    

#### 4.3.1. Front-End
The Mojolicious Web Framework runs on top of a perl instance, which serves the back-end requests and passes back and forth json, as well as the ui Mojo templates dynamically, which combined with the vue template create the generic ui. 

    

#### 4.3.2. Back-End
The id's of the tables which ARE VISIBLE to the end users ui are big integers, which are formed by the concatenation of the year, month, day, hour, minutes and second in which the row in the table is created. 

    

### 4.4. Hot meta data reload functionality
In most of the modern applications using databases as main data storage, the changes in the Entity models mean in most of the cases full release of the application. Qto simplistic UI interfaces and the hot meta-data reload functionality enable all the changes to the db, which do not break the qto table api ( basically guid and id columns), to be performed only without even having to restart the application layer, but only by reloading one of the following meta-data tables - items_doc, app_items, app_item_attributes

    

## 5. FULL STACK DESCRIPTION


    

### 5.1. OS stack
The Qto Application supports ONLY Ubuntu 18.04 as of v0.7.9. - that is all the deployment automations have been tested ONLY against this OS. 
Any middle level full stack developer should be able to deploy and operate the Qto Application on most of the reason Linux distributions and even probably BSD, yet this will require substantial rewrite of the existing deployment scripts as the aim is to have single code base for those scripts and configurability if multiple OS's will be supported.

    

### 5.2. The Application Layer stack


    

#### 5.2.1. Perl
The Application Layer is written in Perl, which is the best possible return on invested time suiting for the philosophy of the qto application. 

    

#### 5.2.2. The perl modules


    

### 5.3. The Mojolicious web framework
Mojolicious is non-arguably the best web-framework out there in almost any criteria, most notably speed, stability, deployment time, scalability, security and tons of other features you can read about in the framework's web page @: https://mojolicious.org/

    

### 5.4. The browser run-time
Qto support all modern browsers. Internet Explorer / other Microsoft browsers ARE NOT supported, due to their notorious time-burning compliance with web standards. 
Mobile browsers albeit supported are NOT priority ( for now ).

    

#### 5.4.1. HTML 5


    

#### 5.4.2. JavaScript 


    

##### 5.4.2.1. Vue


    

## 6. APPLICATION CONTROL FLOW 
This section provides a generic control flow description for the shell based and ui based control flows. 

    

### 6.1. Shell control flow
The shell control flow is based on the control model input output architecture. The usual pattern is to call a single &lt;&lt;doSomeAction&gt;&gt; shell function, which is stored in a some-action.func.sh file and loaded dynamically based on this naming convention. Maximun of 2 level nesting is used in the functional calls, that is once an shell action is evoked it could call only 1 funtion, which might or might be not a shell action function , BUT not more, to address unneeded complexity written in bash. 

    

#### 6.1.1. Front-End
The Mojolicious Web Framework runs on top of a perl instance, which serves the back-end requests and passes back and forth json, as well as the ui Mojo templates dynamically, which combined with the vue template create the generic ui loaded in modern html 5, web sockets and ajax capable browsers.
That said the html,js , vue code is mixed with some perl Mojolicious template code several times per file unit and the logic of building the file units into html pages in defined by the Mojolicious templating system, but once one could grasp the concept the developing of the front-end is more or less html,javascript , vue centric. 

    

#### 6.1.2. Back-End
The id's of the tables which ARE VISIBLE to the end users ui are big integers, which are formed by the concatenation of the year, month, day, hour, minutes and second in which the row in the table is created. 
The primary keys are however GUID's to provide the underlying expandability for cross-domain, cross-db data transfers. 

    

## 7. SECURITY
This section provides an overview of the security of a system operating the qto application. 

    

### 7.1. Non-security mode
In the non-security mode the application does NOT authenticate any one. Both run over https and http. Of course if you use http all the traffic will be in plain text ...

    

### 7.2. Simple native security mode
In this mode the authentication is performed against the project defined in the login page ( where project is actually the database to which the application layer can connect to ). This means that one user can have access to multiple project databases and be authenticated agains some of them thus being able to navigate with the same browser from project to project. In this mode the user credentials - email and password are stored in the users table with a blowfish encryption.
Sessions are used for storing the state of the authentication, which is handled by the Mojolicious web framework - all data gets serialized to json and stored Base64 encoded on the client-side, but is protected from unwanted changes with a HMAC-SHA1 signature. 

    

<file_sep>-- \echo 'If necessary, perform -- DROP TABLE IF EXISTS app_imgs CASCADE'

-- \echo '8. Creating the app_imgs table'

CREATE TABLE app_imgs (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , name           varchar (200) NOT NULL DEFAULT 'Image figure title'
    , relative_path  varchar (1000) NOT NULL DEFAULT 'src/perl/qto/public/dat/img/qto/...'
    , http_path      varchar (4000) NOT NULL DEFAULT 'https://raw.githubusercontent.com/YordanGeorgiev/qto/master/doc/img/..'
    , style          varchar (100) NOT NULL DEFAULT 'width: 800px; height: 600x'
    , item_id        bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint)
    , description    varchar (4000) NOT NULL DEFAULT 'Alt text' 
    , item_name      varchar (200) NOT NULL DEFAULT 'item_name...'
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_app_imgs_guid PRIMARY KEY (guid)
    );


CREATE UNIQUE INDEX idx_uniq_app_imgs_id
	ON app_imgs (id);

-- \echo 'List columns of the created table app_imgs'

SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid='public.app_imgs'::regclass
   AND    attnum>0
   AND    NOT attisdropped
   ORDER  BY attnum; 

-- \echo 'Update time on every EXECUTE trigger:'
CREATE TRIGGER trg_set_update_time_on_app_imgs
	BEFORE UPDATE ON app_imgs
	FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
	FROM pg_trigger
	WHERE NOT tgisinternal
	AND tgrelid='app_imgs'::regclass;
<file_sep>#!/usr/bin/env bash 

# v1.2.1
#------------------------------------------------------------------------------
# creates the full package as component of larger product platform
#------------------------------------------------------------------------------
doCreateFullPackage(){

   test -z ${qto_project:-} && qto_project=qto
	#define default vars
	test -z ${include_file:-}         && \
		include_file="$PRODUCT_DIR/met/.$ENV.$RUN_UNIT"

	# relative file path is passed turn it to absolute one 
	[[ $include_file == /* ]] || include_file=$PRODUCT_DIR/$include_file

   if [ ! -f "$include_file" ]; then
      do_log "FATAL the deployment file: $include_file does not exist !!!"
      return 1
   fi

   tgt_ENV=$(echo `basename "$include_file"`|cut -d'.' -f2)

	# start: add the perl_ignore_file_pattern
	while read -r line ; do \
		got=$(echo $line|perl -ne 'm|^\s*#\s*perl_ignore_file_pattern\s*=(.*)$|g;print $1'); \
		test -z "$got" || perl_ignore_file_pattern="$got|${perl_ignore_file_pattern:-}" ;
	done < <(cat $include_file)

	# or how-to remove the last char from a string 	
	perl_ignore_file_pattern=$(echo "$perl_ignore_file_pattern"|sed 's/.$//')
	test -z $perl_ignore_file_pattern && perl_ignore_file_pattern='.*\.swp$|.*\.log|$.*\.swo$'
	echo perl_ignore_file_pattern::: $perl_ignore_file_pattern
	# note: | egrep -v "$perl_ignore_file_pattern" | egrep -v '^\s*#'
	cd $product_base_dir

	timestamp=`date "+%Y%m%d_%H%M%S"`
	# the last token of the include_file with . token separator - thus no points in names
	zip_file_name=$(echo $include_file | rev | cut -d'.' -f 1 | rev)
   test $zip_file_name != $RUN_UNIT && zip_file_name="$zip_file_name"'--'"$qto_project"
	zip_file_name="$zip_file_name.$product_version.$tgt_ENV.$timestamp.$host_name.zip"
	zip_file="$product_dir/$zip_file_name"
	mkdir -p $PRODUCT_DIR/dat/$RUN_UNIT/tmp
	echo $zip_file>$PRODUCT_DIR/dat/$RUN_UNIT/tmp/zip_file

   cd $product_base_dir; cd .. ;
	# zip MM ops
	# -MM  --must-match
	# All  input  patterns must match at least one file and all input files found must be readable.
	set -x ; ret=1
	cat $include_file | egrep -v "$perl_ignore_file_pattern" | sed '/^#/ d' | perl -ne 's|\n|\000|g;print'| \
	xargs -0 -I "{}" zip -MM $zip_file "${org_name}/$RUN_UNIT/$environment_name/{}"
	ret=$? 
	set +x
	test $ret -gt 0 && (
		while IFS='' read f ; do (
			test -d "$PRODUCT_DIR/$f" && continue ; 
			test -f "$PRODUCT_DIR/$f" && continue ; 
			test -f "$PRODUCT_DIR/$f" || do_log 'ERROR not a file: "'"$f"'"' ;  
			test -f "$PRODUCT_DIR/$f" || ret=1 && exit 1
		); 
		done < <(cat $include_file | egrep -v "$perl_ignore_file_pattern" | sed '/^#/ d')
	);

   if [ ! $ret -eq 0 ]; then
      do_log "FATAL deleted $zip_file , because of packaging errors $! !!!"
	   rm -fv $zip_file
      return 1
   fi
 
   test -z ${mix_data_dir:-} && mix_data_dir=$PRODUCT_DIR/dat/mix
   # backup the project data dir if not running on the product itself ...
   test -d $mix_data_dir/$(date "+%Y")/$(date "+%Y-%m")/$(date "+%Y-%m-%d") || doIncreaseDate
   # and zip the project data dir
   if [ ! $RUN_UNIT == $qto_project ]; then
      cd $mix_data_dir
      for i in {1..3} ; do cd .. ; done ;
      zip -r $zip_file $qto_project/dat/mix/$(date "+%Y")/$(date "+%Y-%m")/$(date "+%Y-%m-%d")
	   cd $org_base_dir
   else 
      zip -r $zip_file $org_name/$RUN_UNIT/$environment_name/dat/mix/$(date "+%Y")/$(date "+%Y-%m")/$(date "+%Y-%m-%d")
   fi

   msg="created the following full development package:"
   do_log "INFO $msg"
   msg="`stat -c \"%y %n\" $zip_file`"
   do_log "INFO $msg"

   if [[ ${network_backup_dir+x} && -n $network_backup_dir ]] ; then
      if [ -d "$network_backup_dir" ] ; then
         doRunCmdAndLog "cp -v $zip_file $network_backup_dir/"
         msg=" with the following network dir backup :
         ""$(stat -c "%y %n" "$network_backup_dir/$zip_file_name")"
         do_log "INFO $msg"
      else
         msg="skip backup as network_backup_dir does not exist"
         do_log "ERROR $msg"
      fi
   else
      msg="skip the creation of the network backup as no network_backup_dir is configured"
      do_log "INFO  $msg"
   fi

   do_log "INFO STOP  create-full-package.func.sh" 

}
#eof func doCreateFullPackage

<file_sep># src/bash/qto/funcs/db-to-xls.func.sh

# v0.6.9
doDbToXls(){

	do_log "DEBUG START doDbToXls"
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!
   do_log "INFO START testing db-to-xls"
   test -z ${tables+x} && export tables='daily_issues'
   perl src/perl/qto/script/qto.pl --do db-to-xls --tables $tables
   exit_code=$?
   do_log "INFO doRunQto exit_code $exit_code"
   test $exit_code -ne 0 && do_exit $exit_code "failed to run qto.pl"  


	do_log "DEBUG STOP  doDbToXls"
}


# eof file: src/bash/qto/funcs/db-to-xls.func.sh
<file_sep># src/bash/lp_nettilaskuri/funcs/generate-action-files.test.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doTestGenerateActionFiles comments ...
# ---------------------------------------------------------
doTestGenerateActionFiles(){

	do_log "DEBUG START doTestGenerateActionFiles"
	
	cat doc/txt/lp_nettilaskuri/tests/generate-action-files.test.txt
	
	doSpecGenerateActionFiles

	doHelpGenerateActionFiles
	
	bash src/bash/lp_nettilaskuri/lp_nettilaskuri.sh -a generate-action-files

	do_log "DEBUG STOP  doTestGenerateActionFiles"
	
	sleep $sleep_interval
	
	source $PRODUCT_DIR/lib/bash/funcs/flush-screen.sh
	do_flush_screen

}
# eof func doTestGenerateActionFiles


# eof file: src/bash/lp_nettilaskuri/funcs/generate-action-files.test.sh
<file_sep>process_help_cmd_args () {
  while [[ $# -gt 0 ]]; do
    case "$1" in
       -h|-help) usage; exit 1 ;;
    -v|-verbose) verbose=1 && shift ;;
      -d|-debug) debug=1 && addSbt "-debug" && shift ;;
              *) addResidual "$1" && shift ;;
    esac
  done
}


# src: https://certbot.eff.org/lets-encrypt/ubuntubionic-other
# the install is in the bootstrap
sudo certbot certonly --webroot

sudo openssl pkey -in /etc/letsencrypt/live/qto.fi/privkey.pem -out \
   /etc/letsencrypt/live/qto.fi/privkey.key

sudo openssl x509 -outform der -in /etc/letsencrypt/live/qto.fi/fullchain.pem -out \
   /etc/letsencrypt/live/qto.fi/fullchain.crt


<file_sep>#
# ---------------------------------------------------------
# cat cnf/qto.dev.host-name.cnf
# [MainSection]
# postgres_app_db     = dev_qto
# postgres_rdbms_host     = host-name
# 
# call by: doParseCnfEnvVars cnf/qto.dev.host-name.cnf
# ---------------------------------------------------------
doParseCnfEnvVars(){

   cnf_file=$1;shift 1;

   test -z "$cnf_file" && echo " you should set the cnf_file !!!"

   cnf_dir=$(perl -e 'use File::Basename; use Cwd "abs_path"; print dirname(abs_path(@ARGV[0]));' -- "$cnf_file")
   PRODUCT_DIR=${cnf_dir%/*}

   INI_SECTION=MainSection

	( set -o posix ; set ) | sort >~/vars.before

	eval `sed -e 's/[[:space:]]*\=[[:space:]]*/=/g' \
		-e 's/#.*$//' \
		-e 's/[[:space:]]*$//' \
		-e 's/^[[:space:]]*//' \
		-e "s|%ProductInstanceDir%|${PRODUCT_DIR}|" \
		-e "s/^\(.*\)=\([^\"']*\)$/export \1=\"\2\"/" \
		< $cnf_file \
		| sed -n -e "/^\[$INI_SECTION\]/,/^\s*\[/{/^[^#].*\=.*/p;}"`

   # and post-register for nice logging
   ( set -o posix ; set ) | sort >~/vars.after

   echo "INFO added the following vars from section: [$INI_SECTION]"
   comm -3 ~/vars.before ~/vars.after | perl -ne 's#\s+##g;print "\n $_ "'
}

#
# ---------------------------------------------------------
# call by: doParseIniEnvVars sfw/sh/isg-pub/isg-pub.mini-nz.ysg-ip-172-31-18-13.conf
#; file: mini-nz.sh.hostname.conf docs at the end
#[MainSection]
#; the name of the project
#project=mini-nz
#;
#; the alias of the project - used for the logical html link and the db names
#; eof file: mini-nz.sh.hostname.conf
#proj_alias=mini_nz
# ---------------------------------------------------------
doParseIniEnvVars(){
   ini_file=$1;shift 1;
   #debug ok   echo ini_file:: $ini_file
   #debug ok   sleep 2
   test -z "$ini_file" && ini_file="$component_dir/$component_name.$host_name.conf"
   test -f "$ini_file" || \
      cp -v $component_dir/$component_name.host_name.conf \
            $component_dir/$component_name.$host_name.conf
   eval `sed -e 's/[[:space:]]*\=[[:space:]]*/=/g' \
      -e 's/;.*$//' \
      -e 's/[[:space:]]*$//' \
      -e 's/^[[:space:]]*//' \
      -e "s/^\(.*\)=\([^\"']*\)$/export \1=\"\2\"/" \
      < $ini_file \
      | sed -n -e "/^\[MainSection\]/,/^\s*\[/{/^[^;].*\=.*/p;}"`
}
#eof func doParseIniEnvVars
<file_sep># src/bash/qto/funcs/generate-sql.test.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doTestGenerateSQL comments ...
# ---------------------------------------------------------
doTestGenerateSQL(){

	do_log "DEBUG START doTestGenerateSQL"
	
	cat doc/txt/qto/tests/generate-sql.test.txt
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!
   bash src/bash/qto/qto.sh -a generate-sql

	do_log "DEBUG STOP  doTestGenerateSQL"
}
# eof func doTestGenerateSQL


# eof file: src/bash/qto/funcs/generate-sql.test.sh
<file_sep>#how-to add a linux group
export group=staff
export gid=20001
sudo groupadd -g "$gid" "$group"
sudo cat /etc/group | grep --color "$group"

export user=phz
export uid=20001
export home_dir=/home/$user
export desc="the qto user"
#how-to add an user
sudo useradd --uid "$uid" --home-dir "$home_dir" --gid "$group" \
--create-home --shell /bin/bash "$user" \
--comment "$desc"
sudo cat /etc/passwd | grep --color "$user"
groups "$user"


# modify a user
usermod -a -G $group $user

# change the password for the specified user (own password)
passwd $user 
#how-to forces to change password when logging in for the first time
passwd -f login 
#change user pass to expire never 
chage -I -1 -m 0 -M 99999 -E -1 $user
# and check results 
chage -l $user


#Ei should not return anything !!!
passwd -s -a | grep NP (=No Password)

#delete an user
userdel $user
#administer the /etc/group file
gpasswd: 
<file_sep># src/bash/qto/funcs/create-ctags.test.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doTestCreateCtags comments ...
# ---------------------------------------------------------
doTestCreateCtags(){

	do_log "DEBUG START doTestCreateCtags"
	
	cat doc/txt/qto/tests/create-ctags.test.txt
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!

	do_log "DEBUG STOP  doTestCreateCtags"
}
# eof func doTestCreateCtags


# eof file: src/bash/qto/funcs/create-ctags.test.sh
<file_sep>-- file: src/sql/pgsql/qto/app-itms/001.create-table.app_users.sql
-- v0.8.4

-- \echo 'If necessary, perform -- DROP TABLE IF EXISTS app_users'

-- \echo '1. Creating the app_users table'

CREATE TABLE public.app_users (
    guid UUID DEFAULT public.gen_random_uuid() NOT NULL,
    id bigint DEFAULT (to_char(CURRENT_TIMESTAMP, 'YYMMDDHH12MISS'::text))::bigint NOT NULL,
    name character varying(100) DEFAULT 'User name'::character varying NOT NULL,
    first_name character varying(50),
    last_name character varying(50),
    email character varying(200) DEFAULT 'Email'::character varying NOT NULL,
    status smallint DEFAULT 1 NOT NULL,
    description character varying(200) DEFAULT ''::character varying,
    password character varying(200) DEFAULT '<PASSWORD>'::character varying NOT NULL,
    update_time timestamp without time zone DEFAULT date_trunc('second'::text, now()),
    CONSTRAINT pk_app_users_guid PRIMARY KEY (guid),
    UNIQUE(email)
);



--
-- TOC entry 3348 (class 0 OID 176764)
-- Dependencies: 281

COPY public.app_users (guid, id, name, first_name, last_name, email, status, description, password, update_time) FROM stdin;
2660a6e9-9e6b-4faa-8264-27a92872657b	190707231513	tau	anonymous	 user	<EMAIL>	1	Test anonymous	{CRYPT}$2a$08$iAmq3xMI4452eOmbexOXFOzccG7/kDVri21RZIainW2kYXq57xbdG	2020-05-01 21:06:10
2660a6e9-9e6b-4faa-8264-27a92872657c	200107231510	trd	test reader	user	<EMAIL>	1	Test reader	{CRYPT}$2a$08$/Z3BoSd2cOO1Enb4xckj9Ocl/8dWGzUxlyaI0fDLveDSEPHQh6XiG	2020-05-01 21:06:10
2660a6e9-9e6b-4faa-8264-27a92872657d	200107231511	ted	test editor	user	<EMAIL>	1	Test editor	{CRYPT}$2a$08$cbAMAMtbNJthEfglhyj6buI4GL13Yia4QTGIZXFE.9jhuVWZ1p/Ru	2020-05-01 21:06:10
9426c566-0abd-4028-830f-82ed9f91d262	200502001502	pio	product instance	owner user	<EMAIL>	1	Test product instance owner	{CRYPT}$2a$08$nTM2/kqvQkNS.vuGV.0bG.h67j8Ci1DreZ16KeVPA8N9SwBohv5cm	2020-05-01 21:15:41
\.
                               
-- \echo 'List columns of the created table app_users'

SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid='public.app_users'::regclass
   AND    attnum>0
   AND    NOT attisdropped
   ORDER  BY attnum; 

-- \echo 'Update time on every EXECUTE trigger:'
CREATE TRIGGER trg_set_update_time_on_app_users
	BEFORE UPDATE ON app_users 
    FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
	FROM pg_trigger
	WHERE NOT tgisinternal
	AND tgrelid='app_users'::regclass;

<file_sep>#
# the following db dump in the s3 of the aws credentials owning user should exist:
# https://s3-$AWS_DEFAULT_REGION.amazonaws.com/$bucket/"$ENV"_$RUN_UNIT"'.latest.insrts.dmp.sql
#
doLoadDbDataFromS3(){

   test -z "${PROJ_INSTANCE_DIR-}" && PROJ_INSTANCE_DIR="$PRODUCT_DIR"
   #
   do_export_json_section_vars $PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json '.env.db'
   do_export_json_section_vars $PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json '.env.aws'

   mkdir -p $PRODUCT_DIR/dat/sql/

   echo "fetch the db inserts data from the s3 bucket from the following url: "
   echo "https://s3-$AWS_DEFAULT_REGION.amazonaws.com/$bucket/"$ENV"_$RUN_UNIT"'.latest.insrts.dmp.sql'
   wget -O "$PRODUCT_DIR/dat/sql/"$ENV'_'$RUN_UNIT".latest.insrts.dmp.sql" \
      "https://s3-$AWS_DEFAULT_REGION.amazonaws.com/$bucket/"$ENV"_$RUN_UNIT"'.latest.insrts.dmp.sql'

   # configure psql to access the db of THIS instance
   PGPASSWORD=${postgres_sys_usr_admin_pw:-} psql -v -t -X -w -U ${postgres_sys_usr_admin:-} \
      --port $postgres_rdbms_port --host $postgres_rdbms_host -d $postgres_app_db < \
         "$PRODUCT_DIR/dat/sql/$ENV"'_'"$RUN_UNIT"'.latest.insrts.dmp.sql'

   test -f $PRODUCT_DIR/dat/tmp/bootstrapping && rm -v $PRODUCT_DIR/dat/tmp/bootstrapping
   test -f $PRODUCT_DIR/bootstrapping && rm -v $PRODUCT_DIR/bootstrapping

   # grant the needed select
   PGPASSWORD="${postgres_sys_usr_admin_pw:-}" psql -v -q -t -X -w -U "${postgres_sys_usr_admin:-}" \
      -h ${postgres_rdbms_host:-} -p ${postgres_rdbms_port:-} -v ON_ERROR_STOP=1 \
      -d ${postgres_app_db:-} \
      -c "GRANT SELECT,INSERT,UPDATE,DELETE,TRUNCATE ON ALL TABLES IN SCHEMA public TO $postgres_app_usr;
         GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO $postgres_app_usr"

   echo "Databases successfully filled with data."

}
<file_sep>-- remove REFERENCES userstories_doc(guid) if necessary
CREATE TABLE rep_userstories_doc (
      guid           UUID NOT NULL DEFAULT gen_random_uuid() REFERENCES userstories_doc(guid)
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , status         varchar (20) NOT NULL DEFAULT 'status...'
    , prio           integer NOT NULL DEFAULT 1
    , name           varchar (200) NOT NULL DEFAULT 'name/title...'
    , description    varchar (4000)
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_rep_userstories_doc_guid PRIMARY KEY (guid)
)
WITH (
	OIDS=FALSE
);<file_sep>#!/usr/bin/env bash 
# file: src/bash/qto/test-qto.sh

umask 022    ;

# print the commands
# set -x
# print each input line as well
# set -v
# exit the script if any statement returns a non-true return value. gotcha !!!
# set -e
trap 'do_exit $LINENO $BASH_COMMAND; exit' SIGHUP SIGINT SIGQUIT
trap "exit $exit_code" TERM
export TOP_PID=$$

# v1.2.8 
#------------------------------------------------------------------------------
# the main function called
#------------------------------------------------------------------------------
main(){
  	doInit
  	doSetVars
	doRunTests "$@"
  	do_exit 0 "# = STOP  MAIN = $RUN_UNIT_TESTER "

}
#eof main


# v1.2.8 
#------------------------------------------------------------------------------
# the "reflection" func
#------------------------------------------------------------------------------
get_function_list() {
    env -i bash --noprofile --norc -c '
    source "'"$1"'"
    typeset -f |
    grep '\''^[^{} ].* () $'\'' |
    awk "{print \$1}" |
    while read -r function_name; do
        type "$function_name" | head -n 1 | grep -q "is a function$" || continue
        echo "$function_name"
    done
'
}
#eof func get_function_list


#
# v1.2.8
#------------------------------------------------------------------------------
# run all the actions
#------------------------------------------------------------------------------
doRunTests(){
   cd $PRODUCT_DIR

   do_logTestRunEntry 'INIT'
   do_log "INFO actions list file:
   $PRODUCT_DIR/src/bash/qto/tests/run-qto-tests.lst"
   do_log "INFO running the following actions : "
   cat $PRODUCT_DIR/src/bash/qto/tests/run-qto-tests.lst | grep -v '#'
   sleep 1

	while read -r action ; do (

		# exit as soon as ## is found
		[[ ${action:0:2} = \#\# ]] && break
		# do not run a test if it is commented out ( starts with # )
		[[ ${action:0:1} = \# ]] && continue

		do_log "INFO START :: testing action: \"$action\""
		do_logTestRunEntry 'START'
		while read -r test_file ; do (
			# do_log "test_file: \"$test_file\""
			while read -r function_name ; do (
				action_name=`echo $(basename $test_file)|sed -e 's/.test.sh//g'`
            
            if [ "$action_name" != "$action" ]
            then
               continue
            else
				   do_log "INFO START ::: calling function":"$function_name"
               do_logTestRunEntry 'INFO' ' '$(date "+%H:%M:%S")
               $function_name
               do_log "INFO test-qto loop exit_code: $exit_code" 
               do_logTestRunEntry 'INFO' ' '$(date "+%H:%M:%S")
               do_logTestRunEntry 'INFO' ' '"$action_name"
               # all testing functions should export their exit code
               do_logTestRunEntry 'STOP' $exit_code
               # and clear the screen
               do_log "INFO STOP :: testing action: \"$action\""
               # test $exit_code -ne 0 && do_exit $exit_code "FATAL $function_name"
				   sleep $sleep_interval
				   do_flush_screen
            fi
			);
			done< <(get_function_list "$test_file")
		); 
		done < <(find src/bash/qto/tests -type f -name '*.sh')

	);
	done < <(cat $PRODUCT_DIR/src/bash/qto/tests/run-qto-tests.lst)

   do_logTestRunEntry 'STATUS'
}
#eof fun doRunTests


# v1.2.8 
#------------------------------------------------------------------------------
# register the run-time vars before the call of the $0
#------------------------------------------------------------------------------
doInit(){
   call_start_dir=`pwd`
   wrap_bash_dir=$(perl -e 'use File::Basename; use Cwd "abs_path"; print dirname(abs_path(@ARGV[0]));' -- "$0")
   tmp_dir="$wrap_bash_dir/tmp/.tmp.$$"
   mkdir -p "$tmp_dir"
   ( set -o posix ; set ) | sort >"$tmp_dir/vars.before"
   my_name_ext=`basename $0`
   RUN_UNIT_TESTER=${my_name_ext%.*}
   host_name=$(hostname -s)
   export sleep_interval=${sleep_interval:=0}   # to slow down during debuging export sleep_iterval=3
}
#eof doInit


# v1.0.8
#------------------------------------------------------------------------------
# clean and exit with passed status and message
#------------------------------------------------------------------------------
do_exit(){
   exit_code=0
   exit_msg="$*"

   echo -e "\n\n"
	cd $call_start_dir

   case $1 in [0-9])
      exit_code="$1";
      shift 1;
   esac

   if (( $exit_code != 0 )); then
      exit_msg=" ERROR --- exit_code $exit_code --- exit_msg : $exit_msg"
      >&2 echo "$exit_msg"
      # doSendReport
      do_log "FATAL STOP FOR $RUN_UNIT_TESTER RUN with: "
      do_log "FATAL exit_code: $exit_code exit_msg: $exit_msg"
   else
      do_log "INFO  STOP FOR $RUN_UNIT_TESTER RUN with: "
      do_log "INFO  STOP FOR $RUN_UNIT_TESTER RUN: $exit_code $exit_msg"
   fi

   doCleanAfterRun

   #src: http://stackoverflow.com/a/9894126/65706
   test $exit_code -ne 0 && kill -s TERM $TOP_PID
   test $exit_code -eq 0 && exit 0

}
#eof func do_exit

# v1.2.8 
#------------------------------------------------------------------------------
# echo pass params and print them to a log file and terminal
# with timestamp and $host_name and $0 PID
# usage:
# do_log "INFO some info message"
# do_log "DEBUG some debug message"
#------------------------------------------------------------------------------
do_log(){
   type_of_msg=$(echo $*|cut -d " " -f1)
   msg="$(echo $*|cut -d " " -f2-)"
   [[ $type_of_msg == DEBUG ]] && [[ $do_print_debug_msgs -ne 1 ]] && return
   [[ $type_of_msg == INFO ]] && type_of_msg="INFO "

   # print to the terminal if we have one
   test -t 1 && echo " [$type_of_msg] `date "+%Y.%m.%d-%H:%M:%S"` [qto][@$host_name] [$$] $msg "

   # define default log file none specified in cnf file
   test -z $log_file && \
		mkdir -p $PRODUCT_DIR/dat/log/bash && \
			log_file="$PRODUCT_DIR/dat/log/bash/$RUN_UNIT_TESTER.`date "+%Y%m"`.log"
   echo " [$type_of_msg] `date "+%Y.%m.%d-%H:%M:%S"` [$RUN_UNIT_TESTER][@$host_name] [$$] $msg " >> $log_file
}
#eof func do_log


#v1.2.8
#------------------------------------------------------------------------------
# cleans the unneeded during after run-time stuff
# do put here the after cleaning code
#------------------------------------------------------------------------------
doCleanAfterRun(){
   # remove the temporary dir and all the stuff below it
   cmd="rm -fvr $tmp_dir"
   doRunCmdAndLog "$cmd"


#   while read -r f ; do 
#      test -f $f && rm -fv "$f" ; 
#   done < <(find "$wrap_bash_dir" -type f -name '*.bak')
}
#eof func doCleanAfterRun


# v1.2.8 
#------------------------------------------------------------------------------
# run a command and log the call and its output to the log_file
# doPrintHelp: doRunCmdAndLog "$cmd"
#------------------------------------------------------------------------------
doRunCmdAndLog(){
  cmd="$*" ;
  do_log "DEBUG running cmd and log: \"$cmd\""

   msg=$($cmd 2>&1)
   ret_cmd=$?
   error_msg=": Failed to run the command:
		\"$cmd\" with the output:
		\"$msg\" !!!"

   [ $ret_cmd -eq 0 ] || do_log "$error_msg"
   do_log "DEBUG : cmdoutput : \"$msg\""
}
#eof func doRunCmdAndLog


# v1.2.8 
#------------------------------------------------------------------------------
# run a command on failure exit with message
# doPrintHelp: doRunCmdOrExit "$cmd"
# call by:
# set -e ; doRunCmdOrExit "$cmd" ; set +e
#------------------------------------------------------------------------------
doRunCmdOrExit(){
   cmd="$*" ;

   do_log "DEBUG running cmd or exit: \"$cmd\""
   msg=$($cmd 2>&1)
   ret_cmd=$?
   # if occured during the execution exit with error
   error_msg=": FATAL : Failed to run the command \"$cmd\" with the output \"$msg\" !!!"
   [ $ret_cmd -eq 0 ] || do_exit "$ret_cmd" "$error_msg"

   #if no occured just log the message
   do_log "DEBUG : cmdoutput : \"$msg\""
}
#eof func doRunCmdOrExit


# v1.2.8 
#------------------------------------------------------------------------------
# set the variables from the $0.$host_name.cnf file which has ini like syntax
#------------------------------------------------------------------------------
doSetVars(){
   cd $wrap_bash_dir
   for i in {1..3} ; do cd .. ; done ;
   export PRODUCT_DIR=`pwd`;
   
   # add the do_logTestRunEntry func
   source "$PRODUCT_DIR/src/bash/qto/funcs/log-test-run-entry.func.sh"
   source $PRODUCT_DIR/lib/bash/funcs/flush-screen.sh

   # include all the func files to fetch their funcs 
   while read -r test_file ; do . "$test_file" ; done < <(find . -name "*test.sh")
   #while read -r test_file ; do echo "$test_file" ; done < <(find . -name "*test.sh")

   while read -r spec_file ; do . "$spec_file" ; done < <(find . -name "*spec.sh")
   #while read -r test_file ; do echo "$test_file" ; done < <(find . -name "*test.sh")
	
   while read -r help_file ; do . "$help_file" ; done < <(find . -name "*help.sh")
   #while read -r test_file ; do echo "$test_file" ; done < <(find . -name "*test.sh")
   
   # this will be dev , tst, prd
   ENV=$(echo `basename "$PRODUCT_DIR"`|cut -d'.' -f5)
   product_version=$(echo `basename "$PRODUCT_DIR"`|cut -d'.' -f2-4)
   environment_name=$(basename "$PRODUCT_DIR")

   cd ..
   product_dir=`pwd`;

   cd ..
   product_base_dir=`pwd`;

   org_dir=`pwd`
   org_name=$(echo `basename "$org_dir"`)

   cd ..
   org_base_dir=`pwd`;

   cd "$wrap_bash_dir/"

   # start settiing default vars
   do_print_debug_msgs=0
   # stop settiing default vars


   doParseConfFile
   ( set -o posix ; set ) | sort >"$tmp_dir/vars.after"


   do_log "INFO # --------------------------------------"
   do_log "INFO # -----------------------"
   do_log "INFO # ===		 START MAIN   === $RUN_UNIT_TESTER"
   do_log "INFO # -----------------------"
   do_log "INFO # --------------------------------------"
		
   exit_code=0
   do_log "INFO using the following vars:"
   cmd="$(comm -3 $tmp_dir/vars.before $tmp_dir/vars.after | perl -ne 's#\s+##g;print "\n $_ "' )"
   echo -e "$cmd"

   # and clear the screen
   do_flush_screen
}
#eof func doSetVars


# v1.2.8
#------------------------------------------------------------------------------
# parse the ini like $0.$host_name.cnf and set the variables
# cleans the unneeded during after run-time stuff. Note the MainSection
# courtesy of : http://mark.aufflick.com/blog/2007/11/08/parsing-ini-files-with-sed
#------------------------------------------------------------------------------
doParseConfFile(){
	# set a default configuration file
	cnf_file="$wrap_bash_dir/$RUN_UNIT_TESTER.cnf"

	# however if there is a host dependant cnf file override it
	test -f "$wrap_bash_dir/$RUN_UNIT_TESTER.$host_name.cnf" \
		&& cnf_file="$wrap_bash_dir/$RUN_UNIT_TESTER.$host_name.cnf"
	
	# if we have perl apps they will share the same configuration settings with this one
	test -f "$PRODUCT_DIR/$RUN_UNIT_TESTER.$host_name.cnf" \
		&& cnf_file="$PRODUCT_DIR/$RUN_UNIT_TESTER.$host_name.cnf"

	# yet finally override if passed as argument to this function
	# if the the ini file is not passed define the default host independant ini file
	test -z "$1" || cnf_file=$1;shift 1;
	#debug echo "@doParseConfFile cnf_file:: $cnf_file" ; sleep 6
	# coud be later on parametrized ... 
	INI_SECTION=MainSection

	eval `sed -e 's/[[:space:]]*\=[[:space:]]*/=/g' \
		-e 's/#.*$//' \
		-e 's/[[:space:]]*$//' \
		-e 's/^[[:space:]]*//' \
		-e "s/^\(.*\)=\([^\"']*\)$/\1=\"\2\"/" \
		< $cnf_file \
		| sed -n -e "/^\[$INI_SECTION\]/,/^\s*\[/{/^[^#].*\=.*/p;}"`
   
		
}
#eof func doParseConfFile



# Action !!!
main "$@"


#
#----------------------------------------------------------
# Purpose:
# a simplistic tester for the qto
#----------------------------------------------------------
#
#----------------------------------------------------------
# Requirements: bash , perl , [ ctags ] , [ mutt ]
#
#----------------------------------------------------------
#
#----------------------------------------------------------
#  EXIT CODES
# 0 --- Successfull completion
# 1 --- required binary not installed 
# 2 --- Invalid options 
# 3 --- deployment file not found
# 4 --- perl syntax check error
#----------------------------------------------------------
#
# VersionHistory:
#------------------------------------------------------------------------------
# 1.2.0 --- 2020-06-23 13:21:22 -- source flush_screen function from lib folder
# 1.1.0 --- 2017-03-05 16:48:13 -- change to do_exit , added testing report
# 1.0.0 --- 2016-09-11 12:24:15 -- init from bash-stub
#----------------------------------------------------------
#
# eof file: src/bash/qto/test-qto.sh
<file_sep># usage: include it in your Makefile by:
# include lib/make/make-help.task

# TODO-ensure that
ifdef $(shell which terraform)
	TERRAFORM_VERSION := $(shell terraform --version | head -n1 | tr -d 'Terraform v')
	TERRAFORM_REQUIRED_VERSION := "1.0.1"
endif


.PHONY: check-terraform ## @-> checks the terraform version
check-terraform:
	@echo "Checking Terraform version... expecting version [${TERRAFORM_REQUIRED_VERSION}], found [${TERRAFORM_VERSION}]"
	@if [ "${TERRAFORM_VERSION}" != "${TERRAFORM_REQUIRED_VERSION}" ]; then \
		echo "Please ensure you are running terraform version ${TERRAFORM_REQUIRED_VERSION}"; \
		exit 1; \
	fi
<file_sep>#  END USER GUIDE
* [1. INTRODUCTION](#1-introduction)
  * [1.1. DOCUMENT STATUS ](#11-document-status-)
  * [1.2. PURPOSE](#12-purpose)
  * [1.3. AUDIENCE](#13-audience)
  * [1.4. MASTER STORAGE AND STORAGE FORMAT](#14-master-storage-and-storage-format)
  * [1.5. SWITCH PROJECTS BY THE :TO OPERATOR IN THE SEARCH-BOX](#15-switch-projects-by-the-to-operator-in-the-search-box)
  * [1.6. VERSION CONTROL](#16-version-control)
* [2. INITIAL CONCEPTS](#2-initial-concepts)
  * [2.1. PROCESSES RELATED TO THIS DOCUMENT](#21-processes-related-to-this-document)
  * [2.2. ACCESSING MULTIPLE APPLICATIONS FROM THE SAME UI](#22-accessing-multiple-applications-from-the-same-ui)
  * [2.3. SWITCH PROJECTS FROM THE SEARCH BOX](#23-switch-projects-from-the-search-box)
  * [2.4. USING THE APPLICATION FROM BOTH MOBILE AND DESKTOP BROWSERS](#24-using-the-application-from-both-mobile-and-desktop-browsers)
* [3. LOGIN AND LOGOUT](#3-login-and-logout)
  * [3.1. LOGIN TO A QTO INSTANCE](#31-login-to-a-qto-instance)
  * [3.2. LOGOUT FROM A QTO INSTANCE](#32-logout-from-a-qto-instance)
  * [3.3. ACCESSING MULTIPLE APPLICATIONS FROM THE SAME UI](#33-accessing-multiple-applications-from-the-same-ui)
  * [3.4. OWN PASSWORD CHANGE](#34-own-password-change)
  * [3.5. SWITCH ITEMS BY USING THE :FOR OPERATOR IN THE SEARCH-BOX](#35-switch-items-by-using-the-for-operator-in-the-search-box)
  * [3.6. IMPORTANT PEACE OF INFORMATION ABOUT YOUR SECURITY](#36-important-peace-of-information-about-your-security)
* [4. UI NAVIGATION ](#4-ui-navigation-)
  * [4.1. BACK BUTTON USAGE IN THE BROWSER](#41-back-button-usage-in-the-browser)
  * [4.2. SWITCH ITEMS BY USING THE :FOR OPERATOR IN THE SEARCH-BOX](#42-switch-items-by-using-the-for-operator-in-the-search-box)
  * [4.3. QTO SUPPORTS SHORTCUTS NAVIGATION](#43-qto-supports-shortcuts-navigation)
* [5. THE LEFT MENU](#5-the-left-menu)
  * [5.1. VIEWING TABLES FROM DIFFERENT PROJECTS ( DATABASES )](#51-viewing-tables-from-different-projects-(-databases-))
  * [5.2. OPEN AND CLOSE FOLDERS IN THE LEFT MENU](#52-open-and-close-folders-in-the-left-menu)
  * [5.3. OPENING AND CLOSING THE LEFT MENU](#53-opening-and-closing-the-left-menu)
* [6. THE LIST PAGE](#6-the-list-page)
  * [6.1. VIEWING THE LIST PAGE](#61-viewing-the-list-page)
    * [6.1.1. Viewing the full content of the items](#611-viewing-the-full-content-of-the-items)
    * [6.1.2. Listing url syntax](#612-listing-url-syntax)
      * [6.1.2.1. The "pick" url param](#6121-the-"pick"-url-param)
      * [6.1.2.2. The "hide" url param](#6122-the-"hide"-url-param)
      * [6.1.2.3. The "with=col-operator-value" filter](#6123-the-"with=col-operator-value"-filter)
      * [6.1.2.4. The "where=col-operator-value" filter](#6124-the-"where=col-operator-value"-filter)
      * [6.1.2.5. Filtering with "like"](#6125-filtering-with-"like")
  * [6.2. SORTING AN ITEM TABLE](#62-sorting-an-item-table)
    * [6.2.1. The 'as' url syntax for printing the listing page](#621-the-'as'-url-syntax-for-printing-the-listing-page)
  * [6.3. QUICK FILTERING AN ITEM TABLE](#63-quick-filtering-an-item-table)
  * [6.4. SETTING THE ITEM TABLE PAGING SIZE](#64-setting-the-item-table-paging-size)
  * [6.5. PAGING - SETTING THE ITEM TABLE'S PAGE NUMBER](#65-paging--setting-the-item-table's-page-number)
  * [6.6. ITEM TABLE PAGING](#66-item-table-paging)
  * [6.7. KEYBOARD USABILITY IN THE LIST PAGE](#67-keyboard-usability-in-the-list-page)
    * [6.7.1. Navigability of the list page with the keyboard](#671-navigability-of-the-list-page-with-the-keyboard)
    * [6.7.2. Focus the quick search box](#672-focus-the-quick-search-box)
    * [6.7.3. Undo the edit on a cell](#673-undo-the-edit-on-a-cell)
    * [6.7.4. Keyboard navigation on the edit form](#674-keyboard-navigation-on-the-edit-form)
  * [6.8. NEW ITEM CREATION (CREATE)](#68-new-item-creation-(create))
    * [6.8.1. Successful execution](#681-successful-execution)
    * [6.8.2. Error handling on list page click create new item action](#682-error-handling-on-list-page-click-create-new-item-action)
  * [6.9. ITEM EDIT (UPDATE)](#69-item-edit-(update))
    * [6.9.1. Form edit](#691-form-edit)
    * [6.9.2. In-line edit ( UPDATE )](#692-in-line-edit-(-update-))
      * [6.9.2.1. Table columns resizing](#6921-table-columns-resizing)
      * [6.9.2.2. Contents for the table's cells](#6922-contents-for-the-table's-cells)
      * [6.9.2.3. Successful execution](#6923-successful-execution)
      * [6.9.2.4. Error handling on db update error](#6924-error-handling-on-db-update-error)
      * [6.9.2.5. Nulls handling](#6925-nulls-handling)
    * [6.9.3. Dropbox edit ( UPDATE )](#693-dropbox-edit-(-update-))
  * [6.10. ITEM DELETION ( DELETE )](#610-item-deletion-(-delete-))
  * [6.11. LIST AS PRINT-TABLE PAGE](#611-list-as-print-table-page)
* [7. THE VIEW DOC PAGE](#7-the-view-doc-page)
  * [7.1. NAVIGATING THE VIEW DOC PAGE](#71-navigating-the-view-doc-page)
    * [7.1.1. Loading the view doc page](#711-loading-the-view-doc-page)
    * [7.1.2. Searching the view doc by quick filtering](#712-searching-the-view-doc-by-quick-filtering)
    * [7.1.3. Navigating trough the document via the TOC right menu item](#713-navigating-trough-the-document-via-the-toc-right-menu-item)
  * [7.2. SMART INTERLINKING ( OR CROSS-REFERENCING ) CONTENT FOR SHARING ITEMS TO ANOTHER USERS](#72-smart-interlinking-(-or-cross-referencing-)-content-for-sharing-items-to-another-users)
  * [7.3. MANAGING THE VIEW DOC DATA WITH THE RIGHT CLICK MENU](#73-managing-the-view-doc-data-with-the-right-click-menu)
  * [7.4. THE RIGHT TABLE OF CONTENTS  - THE "TOC MENU"
THE RIGHT TABLE OF CONTENTS ( TOC ) MENU](#74-the-right-table-of-contents--the-"toc-menu"
the-right-table-of-contents-(-toc-)-menu)
    * [7.4.1. Open & dive to manage only parts of the documents](#741-open-&-dive-to-manage-only-parts-of-the-documents)
    * [7.4.2. Open in list](#742-open-in-list)
    * [7.4.3. Export to md](#743-export-to-md)
    * [7.4.4. Add new parent node ( gamma )](#744-add-new-parent-node-(-gamma-))
    * [7.4.5. Add new sibling node ( gamma )](#745-add-new-sibling-node-(-gamma-))
    * [7.4.6. Add new child node ( gamma )](#746-add-new-child-node-(-gamma-))
    * [7.4.7. Remove node ( gamma )](#747-remove-node-(-gamma-))
* [8. THE SEARCH PAGE](#8-the-search-page)
  * [8.1. CYCLING THOUGH THE SEARCH RESULTS](#81-cycling-though-the-search-results)
  * [8.2. OPENING AN ITEM FROM THE SEARCH RESULT AND EDITING](#82-opening-an-item-from-the-search-result-and-editing)
* [9. THE REPORT PAGE](#9-the-report-page)
  * [9.1. REPORTS VIEWING](#91-reports-viewing)
  * [9.2. REPORTS CREATION](#92-reports-creation)




    

## 1. INTRODUCTION


    

### 1.1. Document status 
This document is updated constantly in every release of the qto. Each version however is considered to be a complete version regarding the qto version it situated in.

    

### 1.2. Purpose
The purpose of this  document is to describe all the features, functionalities and capabilities provided by a properly installed, configured and operated instance of the qto application for the perspective of an application end-user.
This document is the CONTRACT between you as the potential, former or active user of an qto instance and the product owner of that instance. Thus, should something here be described to work in a particular way, contrarian of your user experience, you should create an issue and assign it to your instance owner. 


    

### 1.3. Audience
This document could be of interest for any potential and actual end-users of the qto application.

    

### 1.4. Master storage and storage format
The master storage of this document is the _doc table of the production database of the instance you are using. 

    

### 1.5. Switch projects by the :to operator in the search-box
If you type the ":to &lt;&lt;database-name&gt;&gt;" you will get a drop down which will list the projects databases , to which your instance has access to, by choosing the database from the list and hitting enter you will be redirected to the same url by on the different database.

    

### 1.6. Version control
The contents of this document MUST be updated according to the EXISTING features, functionalities and capabilities of the application version, in which this document residues.
The version of this document is the same as the version of the qto application. Place your mouse over the upper left corner displaying the database name this document is served from. 

    

## 2. INITIAL CONCEPTS
A single qto web application might have access to one or many databases ( called also projects ). 

    

### 2.1. Processes related to this document
The qto provides a mean for tracking of this documentation contents to the source code per feature/functionality, thus should you find inconsistencies in the behaviour of the application and the content of this document you should create a bug issue by clicking on the plus button of the enduser_issues and assign it to the owner of your product instance. 

    

### 2.2. Accessing multiple applications from the same UI
If you type the ":to &lt;&lt;database-name&gt;&gt;" you will get a drop down which will list the projects databases , to which your instance has access to, by choosing the database from the list and hitting enter you will be redirected to the same url by on the different database.

    

### 2.3. Switch projects from the search box
If you type the ":to &lt;&lt;database-name&gt;&gt;" you will get a drop down which will list the projects databases , to which your instance has access to, by choosing the database from the list and hitting enter you will be redirected to the same url by on the different database.

    

### 2.4. Using the application from both mobile and Desktop browsers
Albeit not explicitly designed for mobile usage the qto web application is usable from larger smartphones and especially tablets.

    

## 3. LOGIN AND LOGOUT
You need to send e-mail to the administrator of the qto application ( just try to login with your e-mail and you will get his/her e-mail in the error msg ). 
This section provides initial concepts for new users to quickly grasp the basics of the qto application.
Important you MUST have cookies enabled otherwise you will not be able to use the qto instance ( if that sounded too technical more than 99% of all the browsers have by default their cookies enabled, if you use regular web services such as Google, Facebook, Instagram you probably have your cookies enabled ... )

    

### 3.1. Login to a qto instance
Type the login url of the qto instance you have access to, for example: https://qto.fi/qto/login. Type your email and password and hit enter or click the login button. 
Should you need access the e-mail of the product instance owner - that is the person, who could grant you access will be displayed in the error msg. 
There might be multiple projects accessible from the same qto instance, that is the following urls authenticate against different projects: 
 https://qto.fi/some_project/login
 https://qto.fi/another_project/login

    

### 3.2. Logout from a qto instance
You logout from any qto instance by opening first the left menu from the upper left corner and clicking the logout img. This will clear your session cookie. The qto application DOES NOT plant any tracking cookies, nor use any analytics to save or track your web behaviour. We usually recommend to use a real e-mail address to register to the qto application, and not even provide your first and / or last name simply to protect your privacy. 
If you are using multiple project the logout removes the authentication for ONLY the project you logged out from.

    

### 3.3. Accessing multiple applications from the same UI
If you type the ":to &lt;&lt;database-name&gt;&gt;" you will get a drop down which will list the projects databases , to which your instance has access to, by choosing the database from the list and hitting enter you will be redirected to the same url by on the different database.

    

### 3.4. Own password change
After you have the password from the administrator change it immediately after login from the &lt;&lt;app&gt;&gt;/list/users page. You can see only your own password - once you have updated it it gets stored encrypted in the database and even the user administrator cannot read it in clear text. 


    

### 3.5. Switch items by using the :for operator in the search-box
If you type the ":to &lt;&lt;database-name&gt;&gt;" you will get a drop down which will list the projects databases , to which your instance has access to, by choosing the database from the list and hitting enter you will be redirected to the same url by on the different database.

    

### 3.6. Important peace of information about YOUR Security
Every user except the administrator can see ONLY his/her own user details. Even the instance administrator cannot see your password, but of course he/she can reset it for you.
The qto by itself DOES NOT store any personal details for tracking you on the web, nor will it sell your personal data. Like NEVER! In fact we recommend providing as little personal data as possible - even by not using real personal names.
The qto application users http(s) session to identify you and technically user tracking could have been implemented to gain some user data, yet from the source code of the application AND from the application documentation is evident that any such attempts, have not been planned, done, nor there are any attempts for the near and distant future.

    

## 4. UI NAVIGATION 


    

### 4.1. Back button usage in the browser
The qto is NOT a "modern" Single Page Application - by not doing SPA (pun intended ) we ensure that you can quickly navigate via the browser history back and forth as each page loads quickly and smoothly:
And since the majority of the web users today are using Chrome, you might want to refresh your Chrome browser shortcuts : https://support.google.com/chrome/answer/157179?hl=en to be even faster with qto.

    

### 4.2. Switch items by using the :for operator in the search-box
If you type the ":to &lt;&lt;database-name&gt;&gt;" you will get a drop down which will list the projects databases , to which your instance has access to, by choosing the database from the list and hitting enter you will be redirected to the same url by on the different database.

    

### 4.3. Qto supports shortcuts navigation
For example typing / in a non-editable control ( textbox, textarea ) will always focus the searchbox in qto, you could quickly type a searchable word and hit enter .... 

    

## 5. THE LEFT MENU


    

### 5.1. Viewing tables from different projects ( databases )
Each project in qto is actually stored in it's own database, to the access for example the dev_qto, tst_qto and prd_qto projects( which could be any names, but in this example just happen to be the dev, tst and prd databases for qto ) you should simply add the db name as the first url part:
https://qto.fi:441/dev_qto/list/release_issues
https://qto.fi:442/tst_qto/list/release_issues
https://qto.fi:443/prd_qto/list/release_issues

    

### 5.2. Open and close folders in the left menu
If there are too much items in the left menu you can open and close them.

    

### 5.3. Opening and closing the left menu
You can access the different pages configured for your project from the upper left menu. Click with the mouse and the menu will open a file explorer like menu containing different folders and "documents". The left menu is accessible from all the pages. If you click somewhere else and not in the left menu, or in the X closing "button" at the top of it the left menu will be closed.

    

## 6. THE LIST PAGE
To see a subset of any database table based on filtering criteria in the url of the qto app you would have the following url: 
https://qto.fi/qto/list/yearly_issues_2020?&oa=prio&pg-size=7

The "list-my page" works exactly the same way as the list page, except for the fact that the subset is ALWAYS restricted by the records belonging to the currently logged in user.

The syntax of the list and list-my pages is constructed as follows:



    # https://qto.fi:442/qto/list-my/yearly_issues_2020?&oa=prio&pg-size=7
    protocol      -- https 
    site          -- qto.fi
    port          -- 442
    qto           -- the project db ( migh be also different than qto )
    list-my       -- the action to perform on the project db - in this case list
    yearly_issues -- the subject of the action - in this case - WHAT to link 

### 6.1. Viewing the list page
You can use the url parameter to select for only desired attributes.
You could filter the result the same way the filters for the select page work ( see below ). 
Should there be errors in the loading of the page, they will be displayed in a msg at the top of the page.

    

#### 6.1.1. Viewing the full content of the items
You can quickly view the full content of each cell of the listing table by hovering with the mouse on top of it. Note that all the links in the content are replaced with clickable links in the tooltip as well as the internal links such as the following one : enduser_guide_doc-190214224315 ( which just refer to next item in this document).

    

#### 6.1.2. Listing url syntax
The listing url syntax mimics the sql select clause syntax, yet in much more simplified form. 

    

##### 6.1.2.1. The "pick" url param
You can "pick" particular columns from the item used in the listing page by adding the url parameter pick with the comma separated list of the columns to pick for example: 
https://qto.fi/qto/list/yearly_issues_2020?as=grid&pick=id,name,description&page-size=5&page-num=1

    

##### 6.1.2.2. The "hide" url param
You can hide specific columns from an item listing by using the hide url parameter as follows:
https://qto.fi/qto/list/yearly_issues?hide=description

    

##### 6.1.2.3. The "with=col-operator-value" filter
You can filter the result of the query by using the "with=col-operator-value". The following examples demonstrates, which operators are supported.
An error message is shown if you do not use existing operator. 
The following url demonstrates this syntax:
http://qto.fi/qto/list/yearly_issues_2020?as=grid&pick=id,prio,name&page-size=5&page-num=1&where=prio-eq-1
Note when going to the next page that all the rows' status is 09-done

    with=status-eq-09-done
    list all the items having the attribute "status" equal to the "09-done" string
    
    with=prio-lt-7
    list all the items having the attribute prio smaller than the number 7
    
    this is the list of all the operators supported
           'eq' => '='
         , 'ne' => '<>'
         , 'gt' => '>'
         , 'lt' => '<'
         , 'ge' => '>='
         , 'le' => '<='
         , 'like' => 'like'

##### 6.1.2.4. The "where=col-operator-value" filter
You can filter the result of the query by using the "where=col-operator-value", which works exactly as the with operator, thus the following examples demonstrates, which operators are supported.
An error message is shown if you do not use existing operator. 

    with=status-eq-09-done
    list all the items having the attribute "status" equal to the "09-done" string
    
    with=prio-lt-7
    list all the items having the attribute prio smaller than the number 7
    
    this is the list of all the operators supported
           'eq' => '='
         , 'ne' => '<>'
         , 'gt' => '>'
         , 'lt' => '<'
         , 'ge' => '>='
         , 'le' => '<='
         , 'like' => 'like'

##### 6.1.2.5. Filtering with "like"
The filtering with the like operator translates to the SQL "like" operator- the "like-by=&lt;&lt;attr&gt;&gt;&like-val=&lt;&lt;val&gt;&gt; filtering, where &lt;&lt;attr&gt;&gt; stands for the name of the attribute to use the like operator. 
Example:
https://qto.fi/qto/list/yearly_issues_2020?as=grid&oa=prio&pick=id,prio,name&page-size=5&page-num=1&like-by=name&like-val=add

    

### 6.2. Sorting an item table
The listed table is sortable by clicking on the columns OR by navigating with the tab key on the keyboard on a column and hitting Enter. 
The sorted column is visually shown as the active one on page load:
https://qto.fi/qto/list/yearly_issues_2020?as=grid&oa=prio&pick=id,prio,name&page-size=5&page-num=1&like-by=name&like-val=add

    

#### 6.2.1. The 'as' url syntax for printing the listing page
By default the url syntax of the list page has the "as=grid" default listing format, if you replace it with the "as=print-table" url parameter you will get a bare listing of the data ( all other sorting and paging parameters work as well ) , which you could use for printing as well.

    

### 6.3. Quick filtering an item table
You can filter the already presented part of the result set in the page by using the search textbox. This is only an ui type of filtering for the already loaded data. This type of filtering is different compared to the url parameters filtering by using the with url param syntax and it filters the already fetched from the db data-set, whereas the with=&lt;&lt;attribute&gt;&gt;&lt;&lt;operator&gt;&gt;&lt;&lt;value&gt;&gt; filtering does filter on the database side.
You could focus the quick search textbox by pressing the forward slash on your keyboard. 
The quick search box works instantaneously, thus hitting enter is not needed. 

    

### 6.4. Setting the item table paging size
You can set the page size of the result set to be fetched from the database by using the "&page-size=&lt;&lt;page-size&gt;&gt;" url parameter or by clicking on the page sizes links below the table.
The default and most convenient table paging size is 7, because it allows quick paging of a small result-set without scrolling on the screen ...  

    

### 6.5. Paging - setting the item table's page number
If the result-set requested is larger than the page size you can go to the next page number by using the "&pg-num=&lt;&lt;page-num&gt;&gt;" url parameter. 
You could go to the next page number by clicking on the links just below the quick search textbox. The table control has UI for setting the table page number. 
The pager shows 10 pages at a "pager page" so getting to the end of hundreds of pages ( depending of course on your page size ) is comparably easy. 
You could quickly use the / char shortcut to focus to the quick search box and from there use the tab to quickly navigate to the desired page number. 

    

### 6.6. Item table paging
The table paging is decided by the pg-num=&lt;&lt;page-number&gt;&gt; and the pg-size=&lt;&lt;page-size&gt;&gt; url operators. 

    

### 6.7. Keyboard usability in the list page
The order of all the ui elements of the list-as-grid page has been arranged so that the user could cycle trough the whole page by accessing all the elements quickly.
Power-users will find it extremely convenient to cycle and edit small tables.

    

#### 6.7.1. Navigability of the list page with the keyboard

You can quickly traverse the cells of the table via the tab key, which does go over the non-editable items too ( the id's ) , so that you could quickly scroll the table as scrolling when the editable is in focus does not work. 
The whole table is easily scrollable whenever the cursor is on non-editable part of the table ( the id's column ) and whenever the last rows must be edited the page is scrolled so that the rows are situated in the middle and not the bottom of the screen. 

    

#### 6.7.2. Focus the quick search box
You could focus the quick search by typing / IF you are not editing a cell. Thus the paging on the next cell is quite handy - as you could easily jump onto the quick search and with couple of tabs navigate to the next page. 

    

#### 6.7.3. Undo the edit on a cell
If you were on a cell and types some text without leaving it, but you change your mind you could simply press the Esc key, which will restore the original content of the cell and you could proceed by tab to the next cell. 

    

#### 6.7.4. Keyboard navigation on the edit form
You could open the edit form with the keyboard while your cursors is on the id button of the item in the grid ( Note that the colour of the button text has to change also. By hitting enter the edit form will open witch the title id selected, from there you could cycle with the tab on each control of the form, thus each time the focus leaves a control the data updated or not is saved to the database. You could close the form by hitting enter when the close button is selected. 

    

### 6.8. New item creation (CREATE)
A new item could be added to the table in the ui and thus in the db table by clicking the plus button above the table ( which uses the google material design ui ). 
The new button has a fixed position, thus available during scrolling as well from the same position. 
The new button changes it's appears when focused via the keyboard, and can be pressed when in focus by hitting enter with the keyboard. 
To practice new items' creations and deletions to get comfortable on the app's behaviour you could use the following table: 
https://qto.fi/qto/list/test_create_table?as=grid&pick=id,name&page-size=5&page-num=1&od=id

    

#### 6.8.1. Successful execution
After clicking the plus button the System adds the new row into the database table and presents it into the table ui AS THE FIRST ROW to emphasise the created row - that is the existing sort of the table is changed to the id column. Note that if you had a defined sorting order before the addition of the new item, it has been replaced by the "order by the latest created" sorting order. 

    

#### 6.8.2. Error handling on list page click create new item action
If any error occurs while the creation an error msg is presented clearly with fading effect, which returns the error msg from the database. 
On invalid input the data is not created to the database and nothing is stored. 

    

### 6.9. Item edit (UPDATE)
There are 2 ways to edit an item of the qto application :
 - inline edit
 - form edit

    

#### 6.9.1. Form edit
You open the form to edit an item from the edit button on the left most column in the listing. A modal dialog containing the filled in details of the item appears. You could either simply cycle via the keyboard trough the items, or edit some of the item details, as soon as any of the controls in the form is selected, after leaving the control the data is saved straight to the database. 
To close the edit form either click on the top close button or on the green "ok" button of the bottom of the form.
Note that human readable values from the drop downs are not show, but the guid's of the tables are show ( this behaviour will probably change in the feature ... )

    

#### 6.9.2. In-line edit ( UPDATE )
The grid can be edited inline so that the data is updated to the database. White space in the cells is preserved. 
To practice new items' creations updates and deletions to get comfortable on the app's behaviour please use first the development instances of the qto project: 
https://qto.fi/qto/list/yearly_issues_2020?as=grid&oa=prio&pick=id,prio,name&page-size=5&page-num=1&like-by=name&like-val=add

    

##### 6.9.2.1. Table columns resizing
You can resize the columns of the tables to a max size of 500 pixels by dragging only one text area. Note however that the textarea will NOT be resized bigger than the maximum width of the current column … 
For the visual outlook of the table a certain default values for certain columns' contents widths are assumed.

    

##### 6.9.2.2. Contents for the table's cells
The table's cells should accept any UTF-8 characters including html entities. The textarea's width should adjust automatically till the width of the widest cell in the table column.

    

##### 6.9.2.3. Successful execution
If the single cell inline-edit is successful no msg is presented and the data is updated to the database storage.
If the updated cell was part of the currently sorted column the ui is automatically adjusted to the new sort order ( for example if a numeric sort was applied and the cell had value of 9 with 1..9 range and the smallest to greatest was currently active if the new update is 1 the item will appear in the top of the listing.

    

##### 6.9.2.4. Error handling on db update error
If any error occurs while updating an error msg is presented clearly with fading effect, which returns the error msg from the database. 
On invalid input the data is not updated to the database and the old value in the cell is restored.

    

##### 6.9.2.5. Nulls handling
Nulls handling is somewhat problematic in ui. For now the behaviour by convention is to leave a nullable record in the database as null, whether the cell of the ui table is left empty ( white space chars are also considered empty)

    

#### 6.9.3. Dropbox edit ( UPDATE )
Should a column end with the "_guid" string, by convention it holds a foreign key. By convention the qto displays the select options the values from the name attribute of the primary key table, meaning simply that you could start searching for any of those items by clicking on the x close button first in the dropbox and start searching in the input. Should you search match items from the primary key table you will see them listed and you will be able to choose them either by scrolling with the mouse and hitting enter OR by pointing with the mouse. Once chosen the new value is automatically saved to the database.

    

### 6.10. Item deletion ( DELETE )
You can delete an item by opening the edit modal dialog box and clicking on the delete button in the bottom of the dialog box. Once the modal dialog box closes you will notice that the item has disappeared from the listing and hence the project database.

    

### 6.11. List as print-table page
The list as print-table page is aimed at producing quickly refined result-set from the database for a further copy paste on to another html page ( wiki etc. )  or even print to paper. 
It has all the functionalities as the list as "table" page, without the filtering from the quick search box and without the ui for the pager and page-sizer -the url params for paging and page-sizing work, however. All the url params work as in the grid listing page. 
https://qto.fi/qto/list/yearly_issues_2020?as=grid&oa=prio&pick=id,prio,name&page-size=5&page-num=1&like-by=name&like-val=add&as=print-table

    

## 7. THE VIEW DOC PAGE
The view page presents the data of a database table, having nested-set hierarchical model partially or fully by the means of different controls. The page is shortly called the view-doc page

    

### 7.1. Navigating the view doc page
You can read a document peace by peace ( aka the big elephant ) by clicking on the logical numbers on the left of the titles in the content - the system will always post the click item to the top - note how the address bar in the broser changes too.

    

#### 7.1.1. Loading the view doc page
When the page is loaded all the content of the document is presented / according to the urls params / which are applying the same filtering as in the list page , but on the hierarchical data-set …
Should there be an error a dynamic snackbar is presented with the error message. The snackbar hides itself by default after 3.9 seconds ( set by default , but it could be pinned to the page by the user to view it properly / send it further). 

    

#### 7.1.2. Searching the view doc by quick filtering
Once the page is loaded the top search omnibox is selected. As soon as you start typing the document the ui starts shrinking the page by presenting ONLY the items containing the search string ( the search is case-insensitive). You can cycle and update the content for the displayed items. If your focus is not on an editable element of the page .you could focus the quick search by typing the "/" char similarly to how you could focus the search box in gmail.

    

#### 7.1.3. Navigating trough the document via the TOC right menu item
The Table of Contents ( TOC ) right menu can be opened and closed and scrolled separately from the scroll of the view doc. Whenever you click on the links in the right menu the document content on the left is scrolled nicely with the clicked title to the top of the screen. 
Note that the TOC menu behaves differently than the left menu - it is NOT closing, if you click outside of it. Thus you could edit your documents and keep it open. Should you want to close it use either the closing button on top of it, or the closing button in its bottom.

    

### 7.2. Smart interlinking ( or cross-referencing ) content for sharing items to another users
The items interlinking works both in the list-grid and in the view-doc pages ( the description ).
Any &lt;&lt;item&gt;&gt;-&lt;&lt;id&gt;&gt; where &lt;&lt;id&gt;&gt; is a whole number is converted into the jump to the anchor of that exact item id of that &lt;&lt;item&gt;&gt;. 
For example :
enduser_guide_doc-190214224374 will generate a link to this "items interlinking" title ( Hold on the Ctrl key and click on the link in the tooltip on the right of this paragraph. 
The interlinking works between the different items as well :
For example:
requirements_doc-1
If the item is not a "doc" item - aka not ending with the _doc the link will redirect the user to a filtered table with only this id: 
for example: principles-1805311658

    

### 7.3. Managing the view doc data with the right-click menu
You can right-click ANY of the toc menu items ( including the doc title, which IS NOT right-clickable from the document content) and choose one of the actions to perform on the clicked item:

This feature is in beta mode still - expect some serious quirks along the way ...
If you right-click on a number of a title or on the title in the right TOC listing and choosing one of the following options you could:

    

### 7.4. The right table of contents  - the "TOC menu"
The right Table Of Contents ( TOC ) menu
You open and close the right-click menu from the upper menu icon. The right-click menu presents the structure of the document.You could navigate with the keyboard trough the right menu links too. 
Note that when you click on a link on the right menu the title of the item you clicked on is scrolled to the top of the page.
Right click on the title items presents the context menu containing several different options to perform on a branch of the document. 
Note, that you can quickly navigate with the tab key on the keyboard once the right menu is open and selected ( aka you have to click on it ), you can hit enter once reaching the desired section of the document.

    

#### 7.4.1. Open & dive to manage only parts of the documents
View the current title and it's tree as a separate document altogether - the larger your documents become the easier it could be to edit them once you open just a specific branch from the document as it's own document.

    

#### 7.4.2. Open in list
See the current branch of the document as table, which will display the additional metadata you could define to track your content ( for example status per item ). You could use the open list to quickly add src_code ui element to your items ( for now there is no view doc for this feature ... )

    

#### 7.4.3. Export to md
This option exports this branch as Mark Down document ( GitHub syntax ), Microsoft Azure is supported as well.

    

#### 7.4.4. Add new parent node ( gamma )
Adding new parent node means simply adding a new item in the document on the upper level in the hierarchy as the one from which you right-clicked just after it - thus for example if you right-clicked  the item with number 4.3. which is the 3rd item on the second level from the 4-th item on the first level you will get the 5.0 item and if the 5.0. item already exists it will be set as the 6.0. and so on ...
This feature does does not work 100% ... you MIGHT have to reload the page if the ui does not behave according to the description above, because of its complexity - this is a known bug with WIP status ...

    

#### 7.4.5. Add new sibling node ( gamma )
Adding new sibling node means simply adding a new item in the document on the same hierarchy level as the one from which you right-clicked just after it - thus for example if you right-clicked  the item with number 4.3. which is the 3rd item on the second level from the 4rth item on the first level you will get the 4.4. item and if the 4.4. item already exists it will be set as the 4.5. and so on ...
This feature does does not work 100% ... you MIGHT have to reload the page if the ui does not behave according to the description above, because of its complexity - this is a known bug with WIP status ...

    

#### 7.4.6. Add new child node ( gamma )
Adding new parent node means simply adding a new item in the document on one level below in the hierarchy as the one from which you right-clicked just after it - thus for example if you right-clicked  the item with number 4.3. which is the 3rd item on the second level from the 4rth item on the first level you will get the 4.3.1 item and if the 4.3.1. item already exists it will be set as the 4.3.2. and so on ...
This feature does does not work 100% ... you MIGHT have to reload the page if the ui does not behave according to the description above, because of its complexity - this is a known bug with WIP status ...

    

#### 7.4.7. Remove node ( gamma )
Adding new parent node means simply adding a new item in the document on one level below in the hierarchy as the one from which you right-clicked just after it - thus for example if you right-clicked  the item with number 4.3. which is the 3rd item on the second level from the 4rth item on the first level you will get the 4.3.1 item and if the 4.3.1. item already exists it will be set as the 4.3.2. and so on ...
This feature does does not work 100% ... you MIGHT have to reload the page if the ui does not behave according to the description above, because of its complexity - this is a known bug with WIP status ...

    

## 8. THE SEARCH PAGE
The search page is presented either after you have search globally from either the view page or the list page by typing your search query and hitting enter. Or by navigating to it from the left menu.

    

### 8.1. Cycling though the search results
The search results listing provides a slice of the simple text search from the project database based on the postgres relevance algorithm. If you have more than 7 results you can set the size of the listing, go to the next page of the search result, previous etc. 

    

### 8.2. Opening an item from the search result and editing
To open an item from the search results click on the open button on the item row in the search listing - the dialog box contains the details of the item in read-only - to actually edit this item click on the edit button on the top in the dialog box, you will be redirect to either the list item page or the view doc page ui of this item.

    

## 9. THE REPORT PAGE
You can view the result sets of pre-defined reports in the qto ( which are just postgres rdbms functions returning result sets), for example:
https://qto.fi/qto/report/get_all_users_app_roles



    

### 9.1. Reports viewing
You can use the same paging and quick filtering as in the search and list pages.

    

### 9.2. Reports creation
A technical person having middle level postgres db knowledge and basic understanding of your qto proj-db business domain SHOULD be able to create a report for you ( which a basic select statement in a postgres function ) in about 15 to 30 min , depending on the complexity of the report. Once the report has been created in the db it straight visible in the reports page WITHOUT having to deploy or install anything on the application server.

    

<file_sep>-- v0.8.6
-- \echo 'If necessary, perform -- DROP TABLE IF EXISTS app_items_doc;'
-- \echo '5. Creating the app_items_doc table'

CREATE TABLE app_items_doc (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , seq            integer NULL
    , level          integer NULL
    , lft            bigint  NULL
    , rgt            bigint  NULL
    , type           varchar (10) NOT NULL DEFAULT 'doc'
    , url            varchar (2048) NOT NULL DEFAULT '#' -- 2048 is the absolute max
    , title          varchar (50) NOT NULL DEFAULT 'link-title...'
    , name           varchar (100) NOT NULL DEFAULT 'doc-title...'
    , status         varchar (10) NOT NULL DEFAULT '02-todo'
    , description    varchar (4000)
    , item_id        integer NULL -- the future hook for the table name in the url
    , src            varchar (4000)
    , formats        text NULL
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_app_items_doc_guid PRIMARY KEY (guid)
    );

CREATE unique index idx_uniq_app_items_doc_id
	ON app_items_doc (id);

-- insert into app_items_doc ( id , level , seq , lft , rgt , name) values ( 0 , 0, 1, 1, 8, 'ITEMS DOC' );
-- insert into app_items_doc ( id , level , seq , lft , rgt , name) values ( 1 , 1, 2, 2, 3, 'lists' );
-- insert into app_items_doc ( id , level , seq , lft , rgt , name) values ( 2 , 1, 3, 4, 5, 'docs' );
-- insert into app_items_doc ( id , level , seq , lft , rgt , name) values ( 3 , 1, 4, 6, 7, 'labels' );


-- \echo 'List columns of the created table app_items'
SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid='public.app_items_doc'::regclass
   AND    attnum>0
   AND    NOT attisdropped
   ORDER  BY attnum; 

-- \echo 'Update time on every EXECUTE trigger:'
CREATE TRIGGER trg_set_update_time_on_app_items_doc
	BEFORE UPDATE ON app_items_doc
	FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
	FROM pg_trigger
	WHERE NOT tgisinternal
	AND tgrelid='app_items_doc'::regclass;

--
-- TOC entry 3321 (class 0 OID 46939)
-- Dependencies: 207
-- Data for Name: items_doc; Type: TABLE DATA; Schema: public; Owner: usrtstqtoadmin
--


COPY public.app_items_doc (guid, id, level, seq, type, url, title, name, status, description, item_id, src, formats, lft, rgt, update_time) FROM stdin;
5b9c9736-6d97-44f7-8453-8a63bad5f150	191109112208	2	4	doc	view/readme_doc	readme	readme	09-done	\N	\N	\N	\N	5	6	2020-01-31 18:39:56
7b0e04c7-ccee-4c74-b32c-428916dc8b40	191110102142	2	18	doc	view/concepts_doc	application concepts	concepts	09-done	\N	\N	\N	\N	31	26	2020-03-01 10:29:33
d9aa6513-724c-420f-b694-a583d54564f5	191110102154	2	21	doc	view/features_doc	features & functionalities	features & functionalities	09-done	\N	\N	\N	\N	37	30	2020-03-01 10:29:33
3a0b8f50-d88f-4e54-911e-f9bf84bc7ad1	191110051042	2	23	doc	view/system_guide_doc	system gude	system guide	09-done	\N	\N	\N	\N	41	34	2020-03-01 10:29:33
465b0a5c-13bb-4888-b2d7-6a08d3b86f69	191112032733	2	8	doc	list/ideas?oa=prio	ideas	ideas	09-done	\N	\N	\N	\N	13	12	2020-03-01 10:29:33
6d5b4f93-2694-4b34-8e7a-422da597518c	191113061322	2	8	doc	list/problems?oa=prio	problems	problems	02-todo	\N	\N	\N	\N	15	14	2020-03-01 10:29:33
db69a517-647d-4f68-99bd-7fee79a8de78	200301102933	2	5	doc	view/concepts_doc	concepts	concepts	02-todo	\N	\N	\N	\N	7	8	2020-03-01 10:30:15
e80a74ac-69a4-4aa4-8500-0d8e6029fdfc	191113062934	2	20	doc	view/requirements_doc	requirements	requirements	09-done	\N	\N	\N	\N	35	28	2020-03-01 10:29:33
d6652433-7bee-42c3-a564-0d2580330674	191110044713	2	26	doc	view/devops_guide_doc	devops guide	devops guide	09-done	\N	\N	\N	\N	47	40	2020-03-01 10:29:33
b472356b-4ab2-450d-a444-11f95dbeb20c	191124075545	2	11	doc	list/questions?oa=prio	questions	questions	09-done	\N	\N	\N	\N	19	18	2020-03-01 10:29:33
4763e1ee-abf5-4bd8-a903-9e5bcf64c5bc	191110044834	2	12	doc	list/yearly_issues_2020?&oa=prio&pg-size=7	yearly-issues	yearly_issues	09-done	\N	\N	\N	\N	21	20	2020-03-01 10:29:33
116296a9-23de-47e4-bafd-d0e6366ab892	200301101943	1	13	folder	#	admin corner	admin corner	02-todo	\N	\N	\N	\N	24	31	2020-03-01 10:29:33
dfc0af59-f7e5-46d1-ae5c-43454a755e2b	200301101958	2	14	doc	list/app_items_doc?oa=seq&pick=name,url,title,type	list items doc	list items doc	02-todo	\N	\N	\N	\N	25	24	2020-03-01 10:29:33
f6af2e03-b6b6-41fc-8012-f7f033780b65	200301102020	2	15	doc	view/app_items_doc	view items doc	view items doc	02-todo	\N	\N	\N	\N	27	26	2020-03-01 10:29:33
1edc12ab-f824-44fa-b67d-a08d7eb7d269	200301102057	2	16	doc	list/meta_columns?oa=name	list meta cols	list meta-columns	02-todo	\N	\N	\N	\N	29	28	2020-03-01 10:29:33
5ea1e9e3-246a-4986-8f5b-53f9e8fedb9c	191110044638	2	24	doc	view/installations_doc	installation guide	installation guide	09-done	\N	\N	\N	\N	43	36	2020-03-01 10:29:33
ccabc9fd-39f2-492b-8079-29f58587a72e	191109112117	1	17	folder	#	documents	documents	09-done	\N	\N	\N	\N	30	39	2020-03-01 10:29:33
3a6c7f4d-bf60-4180-9bbe-6f1f8fa2052c	191110043707	1	22	folder	#	devops docs	devops doc store	09-done	\N	\N	\N	\N	40	49	2020-03-01 10:29:33
8d65a710-d998-4fba-bc47-f163a896c378	200319133453	\N	\N	doc	#	link-title...	doc-title...	02-todo	\N	\N	\N	\N	\N	\N	2020-03-19 11:34:51
5960db0b-4508-4c25-9163-5492d6c3245d	200319133458	\N	\N	doc	#	link-title...	doc-title...	02-todo	\N	\N	\N	\N	\N	\N	2020-03-19 11:34:57
18fac44f-48ab-4ce9-98e2-5c953e19a1b0	200319133500	\N	\N	doc	#	link-title...	doc-title...	02-todo	\N	\N	\N	\N	\N	\N	2020-03-19 11:35:46
79599235-ae29-4eb3-8bc9-a4a91989503e	191110044649	2	25	doc	view/maintenance_guide_doc	maintenance & operations guide	maintenance & operations guide	09-done	\N	\N	\N	\N	45	38	2020-03-19 11:36:08
fa4bc87e-8a98-4012-9128-d40c0a15d40b	0	0	1	folder	#	not used	ITEMS DOC	09-done	\N	\N	\N	\N	1	52	2020-03-19 11:36:36
e66250fb-324d-47c8-81f4-25c1ac3dd539	191109110438	1	7	folder	#	lists	lists	09-done	\N	\N	\N	\N	12	23	2020-03-19 11:36:37
fb9c288d-e363-4219-8224-2e6a71e71777	191110045939	2	6	doc	view/enduser_guide_doc	end-user guide	end-user guide	09-done	\N	\N	\N	\N	9	8	2020-03-20 07:37:45
16232847-865e-45b3-90be-21ca1f5636b4	191110044820	2	10	doc	list/monthly_issues_202005?oa=prio	monthly-issues	monthly-issues	09-done	\N	\N	\N	\N	17	16	2020-04-01 02:35:07
3d71a66a-f9e8-45c5-af6b-268cfc671a19	191110050537	2	19	doc	view/userstories_doc	user-stories	userstories doc	09-done	\N	\N	\N	\N	33	28	2020-04-06 14:14:23
c95e8b4f-48a1-4426-8ff3-9c512ee5d21f	1	1	2	folder	#	general	start	09-done	\N	\N	\N	\N	2	11	2020-04-10 17:40:51
8d969e0b-03b4-400c-a8e7-78b723161bd2	191113062550	2	3	doc	search	search	home & search	09-done	\N	\N	\N	\N	3	4	2020-04-10 19:22:32
\.
<file_sep>do_build_docker_image(){

   test -z "${PROJ_INSTANCE_DIR-}" && PROJ_INSTANCE_DIR="$PRODUCT_DIR"

   test -z ${PROJ_CONF_FILE:-} && export PROJ_CONF_FILE="$PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json"
   do_export_json_section_vars $PROJ_CONF_FILE '.env.db'

   test -f "$PRODUCT_DIR/src/docker/Dockerfile.deploy-$RUN_UNIT.$product_version" || \
      do_exit 1 "the src/docker/Dockerfile.deploy-$RUN_UNIT.$product_version cannot be found !!!"

   #doRemoveDockerContainers
   test ${DOCKER_NUCLEAR_OPTION_PURGE:-0} -eq 1 && doFullCleanDocker # obs clears all docker stuff !!!
   #doRemoveDockerImages
   cp -v "$PRODUCT_DIR/src/docker/Dockerfile.deploy-$RUN_UNIT.$product_version" "$PRODUCT_DIR/Dockerfile"

   # for quick tests uncomment ^^^ and use this one :
   # cp -v "$PRODUCT_DIR/src/docker/Dockerfile.deploy-quick-test" "$PRODUCT_DIR/Dockerfile"

   docker build \
   --build-arg PRODUCT_DIR=$PRODUCT_DIR \
   --build-arg postgres_app_db=$postgres_app_db \
   --build-arg TZ=${TZ:-} \
   --build-arg ENV=$ENV \
   --build-arg USER=$USER \
   --build-arg UID=$UID \
   --build-arg GROUP=$GROUP \
   --build-arg GID=$GID \
   --build-arg postgres_app_db=$postgres_app_db \
   --build-arg postgres_sys_usr_admin=$postgres_sys_usr_admin \
   --build-arg root_pwd=$<PASSWORD> \
   --build-arg app_user_pwd=$app_user_pwd \
   -t qto-image:$product_version.$ENV .

   test $? -ne 0 && do_log "FATAL the docker image building failed !!!"
   rm -v "$PRODUCT_DIR/Dockerfile"

   echo -e "\n\n to instantiate a new container, run: \n"
   echo -e "bash $PRODUCT_DIR/src/bash/qto/qto.sh -a run-container\n\n"

}

<file_sep># src/bash/qto/funcs/remove-action-files.help.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doHelpRemoveActionFiles comments ...
# ---------------------------------------------------------
doHelpRemoveActionFiles(){

	do_log "DEBUG START doHelpRemoveActionFiles"
	
	cat doc/txt/qto/helps/remove-action-files.help.txt
	sleep "$sleep_interval"
	# add your action implementation code here ... 

	do_log "DEBUG STOP  doHelpRemoveActionFiles"
}
# eof func doHelpRemoveActionFiles


# eof file: src/bash/qto/funcs/remove-action-files.help.sh
<file_sep># src/bash/qto/funcs/clone-to-app.help.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doHelpCloneToApp comments ...
# ---------------------------------------------------------
doHelpCloneToApp(){

	do_log "DEBUG START doHelpCloneToApp"
	
	cat doc/txt/qto/helps/clone-to-app.help.txt
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!

	do_log "DEBUG STOP  doHelpCloneToApp"
}
# eof func doHelpCloneToApp


# eof file: src/bash/qto/funcs/clone-to-app.help.sh
<file_sep>ENV ?= lde
ifeq ($(ENV),lde)
    # AWS_REGION = eu-central-1
    @echo ENV:$(ENV) in make
endif
ifeq ($(ENV),dev)
    @echo ENV:$(ENV) in make
endif
ifeq ($(ENV),stg)
    @echo ENV:$(ENV) in make
endif
ifeq ($(ENV),prd)
    @echo ENV:$(ENV) in make
endif
<file_sep>               -- DROP TABLE IF EXISTS test_delete_table ; 

SELECT 'create the "test_delete_table" table'
;

   CREATE TABLE test_delete_table (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , name           varchar (200) default 'name ...'
    , description    varchar (4000) default 'desc ...'
    , CONSTRAINT pk_test_delete_table_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

   insert into test_delete_table ( id,name,description) values (1,'name-1','the name should be deleted to deleted-name-1'); 
   insert into test_delete_table ( id,name,description) values (2,'name-2','the name attr should NOT be deleted'); 
   insert into test_delete_table ( id,name,description) values (3,'name-3','the name attr should be deleted to deleted-name-3'); 

SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.test_delete_table'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 


<file_sep>-- DROP TABLE IF EXISTS app_item_attributes CASCADE ; 

SELECT 'create the "app_item_attributes" table'
; 
   CREATE TABLE app_item_attributes (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , prio           integer NOT NULL DEFAULT 1
    , table_name     varchar (100) NOT NULL DEFAULT 'table name ...'
    , name           varchar (100) NOT NULL DEFAULT 'name ...'
    , description    varchar (4000)
    , skip_in_list   boolean null default false
    , width          integer NULL DEFAULT null
    , readonly       boolean null default false 
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    ) WITH (
      OIDS=FALSE
    );

ALTER TABLE app_item_attributes 
  -- DROP CONSTRAINT IF EXISTS uc_table_columns
;
ALTER TABLE app_item_attributes 
  -- DROP CONSTRAINT IF EXISTS pk_app_item_attributes_guid
;
    
ALTER TABLE app_item_attributes 
    ADD CONSTRAINT pk_app_item_attributes_guid PRIMARY KEY (guid)
;
ALTER TABLE app_item_attributes 
    ADD CONSTRAINT uc_table_columns unique (table_name, name)
;

create unique index idx_uniq_app_item_attributes_id on app_item_attributes (id);



SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.app_item_attributes'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 

CREATE TRIGGER trg_set_update_time_on_app_item_attributes BEFORE UPDATE ON app_item_attributes FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
FROM pg_trigger
WHERE NOT tgisinternal
AND tgrelid = 'app_item_attributes'::regclass;

; 

--
-- TOC entry 3322 (class 0 OID 47354)
-- Dependencies: 229
-- Data for Name: meta_columns; Type: TABLE DATA; Schema: public; Owner: usrtstqtoadmin
--

COPY public.app_item_attributes(guid, id, prio, table_name, name, description, skip_in_list, width, readonly, update_time) FROM stdin;
6a89f5fc-1115-4a89-8472-5591a0d50e80	190523215546	4	userstories	update_time	 	t	\N	f	2019-11-22 12:22:06
f36fc822-46a4-49e3-a53a-374036e95bb5	190614191158	4	meta_columns	description	 	f	50	f	2019-12-06 13:00:32
52189bff-d2f9-46b9-b8ba-e2ccd96a6bc7	181206221152	4	userstories	lft_rnk	\N	t	\N	f	2019-11-22 12:13:53
e3f1d0b1-ec58-4bcb-babd-408ae09e8036	181206221206	4	userstories	rgt_rnk	\N	t	\N	f	2019-11-22 12:13:53
aaaeea54-5b28-4f15-ab2b-7dad52f459bd	181125050540	4	monthly_issues	name	\N	f	\N	f	2019-11-22 12:13:53
a982d70f-b9b2-4461-b7d6-549c091c3d61	181125151422	4	meta_columns	update_time	\N	t	\N	f	2019-11-22 12:13:53
c8b49506-8306-48e9-93a2-4789c80411e1	181125104502	4	monthly_issues	start_time	_	t	\N	f	2019-11-22 12:13:53
3f45c255-7168-4369-b786-11ca1c9f5dcd	181125163726	4	monthly_issues	stop_time	\N	t	\N	f	2019-11-22 12:13:53
ace21c06-d7b3-45ae-a4f7-da96291d8050	181125163903	4	monthly_issues	planned_hours	\N	t	\N	f	2019-11-22 12:13:53
106265e2-073d-43a4-81c7-2eb3d0cb8905	181125104525	4	meta_columns	guid	_	t	\N	f	2019-11-22 12:13:53
d95f237e-00e5-4b46-acba-d89ba7099296	181125163942	4	monthly_issues	seq	\N	t	\N	f	2019-11-22 12:13:53
42681e08-31d8-425c-9f17-369561c0ed39	181125165700	4	monthly_issues	level	\N	t	\N	f	2019-11-22 12:13:53
2af06c04-c8b7-4847-8bef-77b77546b520	181125165752	4	monthly_issues	actual_hours	\N	t	\N	f	2019-11-22 12:13:53
153c36a0-b1c4-4bbc-aff0-dfe62688ce6a	181125165803	4	monthly_issues	update_time	\N	t	\N	f	2019-11-22 12:13:53
a4a6ecc0-c43a-4ded-a5cf-d736ec7e9186	181125171301	4	monthly_issues	tags	\N	t	\N	f	2019-11-22 12:13:53
2ffeea85-f182-4290-8862-64f71e26c552	181126131754	4	confs	update_time	\N	t	\N	f	2019-11-22 12:13:53
1aaeea54-5b28-4f15-ab2b-7dad52f459bd	181125150540	4	yearly_issues	name	\N	f	\N	f	2019-11-22 12:13:53
253c36a0-b1c4-4bbc-aff0-dfe62688ce6a	181125065803	4	yearly_issues	update_time	\N	t	\N	f	2019-11-22 12:13:53
a8b49506-8306-48e9-93a2-4789c80411e1	181125004502	4	yearly_issues	start_time	_	t	\N	f	2019-11-22 12:13:53
2f45c255-7168-4369-b786-11ca1c9f5dcd	181125063726	4	yearly_issues	stop_time	\N	t	\N	f	2019-11-22 12:13:53
1ce21c06-d7b3-45ae-a4f7-da96291d8050	181125063903	4	yearly_issues	planned_hours	\N	t	\N	f	2019-11-22 12:13:53
195f237e-00e5-4b46-acba-d89ba7099296	181125063942	4	yearly_issues	seq	\N	t	\N	f	2019-11-22 12:13:53
32681e08-31d8-425c-9f17-369561c0ed39	181125065700	4	yearly_issues	level	\N	t	\N	f	2019-11-22 12:13:53
1af06c04-c8b7-4847-8bef-77b77546b520	181125065752	4	yearly_issues	actual_hours	\N	t	\N	f	2019-11-22 12:13:53
4be7e57e-f306-4afd-98b9-7ba558d67bd2	181125171311	4	yearly_issues	tags	\N	t	\N	f	2019-11-22 12:13:53
c5d174a8-24a7-48ab-b883-8706305f235a	190202220548	4	monthly_issues	weight	\N	t	\N	f	2019-11-22 12:13:53
150ce162-e8a2-4895-835d-9aedc2c8bdf4	190309101902	4	export_files	update_time	 	t	\N	f	2019-11-22 12:13:53
2d519e36-a075-4872-98ed-43b33b6156b7	190501192032	4	problems	seq	 	t	\N	f	2019-11-22 12:13:53
38feebe7-190c-4bf0-b846-384447e7eb81	190708213054	4	release_issues	weight	 	t	0	f	2019-11-22 12:13:53
f4a0db03-fa8a-4e7b-8075-9a0256df4b2b	191122105848	4	release_issues	status	 	f	30	f	2019-11-22 12:13:53
ee7eda74-5a16-42b7-ba7b-ffd363d02bf7	191114190104	4	release_issues	type	The type of an release issue could be: \n - bug , feature , functionality , task , cnf change	f	\N	f	2019-11-22 12:13:53
a6aac2f1-895d-45ee-8b0f-f75ffbc2cca8	191122141244	4	problems	description	 	f	200	f	2019-11-22 12:13:53
70e9d8a4-9060-4bfb-8703-b1924aad7de9	190523213203	4	release_issues	update_time	 	t	\N	f	2019-11-22 12:22:13
3823d020-2fcb-4d5b-ad66-591ed7e3d481	191122142213	1	meta_columns	table_name	 	f	35	f	2019-12-06 11:30:29
cb762fd4-dc5e-4180-b91c-beab27652d37	191122141438	2	problems	update_time	 	t	\N	f	2019-11-22 12:15:22
7bfe04a8-3c45-4a4c-9fb5-eacca9b8a930	191122141418	2	problems	solution_proposal	 	f	200	f	2019-12-06 11:30:30
7784df12-652c-476e-96a1-dc08ac0172e3	191122141515	1	problems	owner	\N	f	10	f	2019-11-22 12:15:32
8421afcc-be0a-4eb1-9a76-77b7dd80f41d	200227200120	1	yearly_issues_2020	update_time	 	t	10	f	2020-02-27 18:01:31
a69e1cc6-2e69-4704-bd62-8b013e8afbc5	200227133517	1	monthly_issues_202004	issues_status_guid	 	f	30	f	2020-04-11 13:36:17
d050bc02-1bcf-4391-ad7b-97146fae662a	191206142026	1	items_doc	url	 	f	40	f	2020-04-11 13:36:22
57c54026-eeaf-468a-8326-9175b174b173	200227191454	1	monthly_issues_202004	category_id	 	f	30	f	2020-04-11 13:37:41
94124d2f-3895-4b6f-966c-1f9bfe42fbec	200227184143	1	items_roles_permissions	roles_guid	 	f	60	f	2020-04-11 13:37:51
\.

<file_sep># src/bash/qto/funcs/run-integration-tests.func.sh

doRunIntegrationTests(){
   test -z "${PROJ_INSTANCE_DIR-}" && PROJ_INSTANCE_DIR="$PRODUCT_DIR"
   #
   do_export_json_section_vars $PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json '.env.db'
   do_export_json_section_vars $PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json '.env.app'

   bash src/bash/qto/qto.sh -a mojo-morbo-stop
   bash src/bash/qto/qto.sh -a mojo-morbo-start
   #test $? -ne 0 && return

   do_log "INFO make a backup of the current db - inserts only "
   bash src/bash/qto/qto.sh -a backup-postgres-db-inserts

   do_log "INFO re-create the $ENV db"
   bash src/bash/qto/qto.sh -a run-pgsql-scripts

   last_db_backup_file=$(find $PROJ_INSTANCE_DIR/dat/mix -name $postgres_app_db*insrts.dmp.sql | sort -n | tail -n 1)
   PGPASSWORD="${postgres_sys_usr_admin_pw:-}" psql -v ON_ERROR_STOP=1 -q -t -X -w \
      -h $postgres_rdbms_host -p $postgres_rdbms_port -U "${postgres_sys_usr_admin:-}" \
      -v postgres_app_db="$postgres_app_db" -f "$last_db_backup_file" -d "$postgres_app_db"
   ret=$? ; test $ret -ne 0 && do_exit 1 "the integration tests failed on db restore .."

   do_log "INFO generating md docs"
   bash src/bash/qto/qto.sh -a generate-md-docs
   test $? -ne 0 && return

   do_log "INFO START integration testing - do run all the implemented action tests"
   perl src/perl/qto/t/TestQto.pl
   test $? -ne 0 && return
	echo -e "\n\n\n"

}

# eof file: src/bash/qto/funcs/run-integration-tests.func.sh
<file_sep>#  README
* [1. WHY](#1-why)
* [2. SO, WHAT IS THIS ?!](#2-so-what-is-this-)
  * [2.1. WHAT CAN IT DO TO FOR ME AND/OR MY ORGANISATION ?](#21-what-can-it-do-to-for-me-and/or-my-organisation-)
  * [2.2. ASSUMPTION AND PREREQUISITES](#22-assumption-and-prerequisites)
  * [2.3. PROPOSED CAPABILITIES](#23-proposed-capabilities)
* [3. DEMO](#3-demo)
* [4. DOCUMENTATION](#4-documentation)
* [5. AUTOMATED DEPLOYMENT LOCALLY AND ON AWS IN 35 MINUTES](#5-automated-deployment-locally-and-on-aws-in-35-minutes)
* [6. ACKNOWLEDGEMENTS](#6-acknowledgements)
* [7. LICENSE](#7-license)




    

## 1. WHY
Why ?! Yet !!  Another App/Tool/Framework/CMS ?! ... ¿¿¿¿¿¿¿ !!!

Software development is prohibitively expensive. In fact any endeavour with team of people aiming towards common goal in any field is expensive, even solely based on persons' time spent. 

If you want to concentrate on your project(s), but have a fast, efficient and simple to use tool to manage it(them), that SIMPLY WORKS, you must read further ...

Still here ?! Let's move on !

    

## 2. SO, WHAT IS THIS ?!
The short and non-technical answer is: 
QTO is a simplistic UI-wise, but extremely scalable and versatile web-based incident management and documentation application used for the managing of multiple projects.
The long and more technically in-depth answer goes as follows: 
A generic and simplistic db centric web-based content management system, build on Postgres CRUDs ( s stands for search ) and hierarchical nested data-sets MULTIPLE databases from the same web application. An included example application is the "qto application", which is kind of incident management and issues handling tool, used for managing of multiple sites and well managing itself ;o). 
The full and extensive https://qto.fi/qto/view/features_doc contains all the features and functionalities of THIS released version. 



Figure: 1 
the 7 main entities of the qto app
![Figure: 1 
the 7 main entities of the qto app](https://raw.githubusercontent.com/YordanGeorgiev/qto/master/doc/img/readme/what-is-is.png)

    

### 2.1. What can it do to for me and/or my organisation ?
Manage your projects with scale and to the finest granularity of your choice by: 
 - incident management
 - issues handling
 - ideas generation
 - official documentation creation ( with interlinked content to those above )
 - &lt;&lt;your-items-of-interest-interlinked&gt;&gt; ( items are just tables with id,and guid ... )
from both mobile and desktop browsers


    

### 2.2. Assumption and prerequisites
Your organisation:
- needs to setup quickly teams and projects for tracking issues, tasks, problems, documentation
- has full trust to the persons per teams for all (except users related data) add, update, delete operations 
- might have the need to save technical documentation in versioned md format
- might have the need to constantly update comparably small ( less than 10k rows) (hierarchy) tables
- needs to keep track vertically of documentation and generate pdf, docx and md files out of it

    

### 2.3. Proposed capabilities
With qto you will gain the following capabilities:
- instance deployment to bare metal/vm install, which should take no more than 40min
- provide db based authentication and restricted access to the app via http(s)
- provide access to the non-technical person via http and/or https for CRUD operations
- quickly define LOTS of tables DDL by using the existing examples and just changing the columns
- search the data from the db via the global search feature 
- load initial data via xls ( less than 10k rows per sheet should be ok )
- provide them with initial links to grasp the "semi-sql" syntax
- create and update multiple hierarchical documents via the view doc interface
- load hierarchical docs via xls, export them to md format ( programatically ) or pdf
- lots of automated sysadmin capabilities ( check the docs in the doc/md dir )

    

## 3. DEMO
You can check the following https://qto.fi/qto/view/enduser_guide_doc of the web app, additionally every doc below has it's "it-doc" link aka the "native" qto document format …
Use the "<EMAIL>" and "secret" credentials to login ( simple click of the login button would do it as well ;o) OR even better try to login with your own e-mail and request access from the admin e-mail displayed to the error msg ...
Still here ?! Interested to get your own instance up-and-running in the cloud exposed to the Internet ?!
Follow simply the instructions in the next section.

    

## 4. DOCUMENTATION

Qto IS about documentation , which are aimed to be as up-to-date to the current release version as possible. Thus you get the following documentation set:
 - readme_doc-0 - the initial landing readme doc for the project
 - userstories_doc-0 - the collection of user-stories used to describe "what is desired"
 - requirements_doc-0 - the structured collection of the requirements 
 - system_guide_doc-0 - architecture and System description
 - devops_guide_doc-0 - a guide for the developers and devops operators
 - installations_doc-0 - a guide for installation of the application
 - enduser_guide_doc-0 - the guide for the usage of the UI ( mainly ) for the end-users
 - concepts_doc-0 - the concepts doc 
Check the doc/md or doc/pdf directories where the generated from the db documents residue in md or pdf format.

    

## 5. AUTOMATED DEPLOYMENT LOCALLY AND ON AWS IN 35 MINUTES
The installation guide https://github.com/YordanGeorgiev/qto/blob/master/doc/md/installations_doc.md contains the instructions on how-to deploy both a local instance AND and an aws instance exposed to the Internet in 35 minutes for instance from scratch on top of the Amazon latest ubuntu 18.04 ami, so that you can have 100% control on your binary configuration on your infrastructure.

    

## 6. ACKNOWLEDGEMENTS
This project would NOT have been possible without the work of the people working on the following frameworks/languages/OS communities listed in no particular order.
 - Perl
 - Mojolicious
 - GNU Linux
 - Vue
 - FreeBSD

Deep gratitudes and thanks to all those people ! This application aims to contain the best practices of our former colleagues and collaborators and fellow travellers in life, which also deserve huge thanks for their support and contributions!!! We tend to incorporate and re-use a lot of code snippets from the Stackoverflow and Codepen sites, should you consider that you were the author of those code snippets and you deserve mentioning of the source please let us know ...

    

## 7. LICENSE
All the trademarks mentioned in the documentation and in the source code belong to their owners. This application uses the Perl Artistic license, check the license.txt file or the following link : https://dev.perl.org/licenses/artistic.html

Should any trademark attribution be missing, mistaken or erroneous, please contact us as soon as possible for rectification.

Usual commercial products licensing conditions usually end-up here, you will get the regular "we do not take responsibility if you mess your data / infrastructure with our code" clause ... - qto is designed from the ground-up differently - read features_doc-191215115117


    

<file_sep>#------------------------------------------------------------------------------
# creates a package from the relative file paths specified in the .env file
#------------------------------------------------------------------------------
doCreateRelativePackage(){

	mkdir -p $product_dir/dat/zip
		test $? -ne 0 && do_exit 2 "Failed to create $PRODUCT_DIR/dat/zip !"

	test -z ${include_file:-}         && \
		include_file="$PRODUCT_DIR/met/.$ENV.$RUN_UNIT"

	# relative file path is passed turn it to absolute one 
	[[ $include_file == /* ]] || include_file=$PRODUCT_DIR/$include_file

	test -f $include_file || \
		do_exit 3 "did not found any deployment file paths containing deploy file @ $include_file"

   tgt_ENV=$(echo `basename "$include_file"`|cut -d'.' -f2)

	timestamp=`date "+%Y%m%d_%H%M%S"`
	# the last token of the include_file with . token separator - thus no points in names
   git_short_hash=$(git rev-parse --short HEAD)
	zip_file_name=$(echo $include_file | rev | cut -d. -f 1 | rev)
	zip_file_name="$zip_file_name.$product_version.$tgt_ENV.$timestamp.$git_short_hash.$host_name.rel.zip"
	zip_file="$product_dir/$zip_file_name"
	

	ret=0
	while read f ; do
		[[ $f == '#'* ]] && continue ; 
		test -d "$PRODUCT_DIR/$f" && continue ; 
		test -f "$PRODUCT_DIR/$f" && continue ; 
		test -f "$PRODUCT_DIR/$f" || do_log 'FATAL cannot find the file: "'"$PRODUCT_DIR/$f"'" to package it' ;  
		test -f "$PRODUCT_DIR/$f" || do_log 'ERROR search for it in the '"$include_file"' ' ;  
		test -f "$PRODUCT_DIR/$f" || do_log 'INFO if you need the file add it to the list file  ' ;  
		test -f "$PRODUCT_DIR/$f" || do_log 'INFO if you do not need the file remove it from the list file  ' ;  
		test -f "$PRODUCT_DIR/$f" || ret=1
		test -f "$PRODUCT_DIR/$f" && break ;
	done < <(cat $include_file)

	do_log "DEBUG ret is $ret "
	test $ret -ne 0 && do_log "ERROR non-existend file specified in the include file: $include_file "
	
	# start: add the perl_ignore_file_pattern
	while read -r line ; do \
		got=$(echo $line|perl -ne 'm|^\s*#\s*perl_ignore_file_pattern\s*=(.*)$|g;print $1'); \
		test -z "$got" || perl_ignore_file_pattern="$got|${perl_ignore_file_pattern:-}" ;
	done < <(cat $include_file)

	# or how-to remove the last char from a string 	
	perl_ignore_file_pattern=$(echo "$perl_ignore_file_pattern"|sed 's/.$//')
	test -z $perl_ignore_file_pattern && perl_ignore_file_pattern='.*\.swp$|.*\.log|$.*\.swo$'
	# note: | grep -vP "$perl_ignore_file_pattern" | grep -vP '^\s*#'


	# zip MM ops -MM = --must-match
	# All  input  patterns must match at least one file and all input files found must be readable.
	ret=0
	cat $include_file | grep -vP $perl_ignore_file_pattern | grep -vP '^\s*#' \
		| perl -ne 's|\n|\000|g;print'| xargs -0 zip -MM $zip_file
   
	ret=$? ; 
   if (( $ret != 0 )); then
      fatal_msg1="deleting $zip_file !!!"
      fatal_msg2="because of packaging errors !!!"
      rm -fv $zip_file
      do_log "FATAL $fatal_msg1"
      do_log "FATAL $fatal_msg2"
      do_exit 1 "FATAL failed to create relative package"
   else
      cd $product_dir
      do_log "INFO created the following relative package:"
      do_log "INFO `stat -c \"%y %n\" $zip_file_name`"

      if [[ ${network_backup_dir+x} && -n $network_backup_dir ]] ; then
         if [ -d "$network_backup_dir" ] ; then
            doRunCmdAndLog "cp -v $zip_file $network_backup_dir/"
            do_log "INFO with the following network backup  :" && \
            do_log "INFO `stat -c \"%y %n\" \"$network_backup_dir/$zip_file_name\"`"
         else
            msg="skip backup as network_backup_dir is not configured"
            do_log "INFO $msg"
         fi
      fi
   fi


}
<file_sep>-- file: src/sql/pgsql/qto/app-itms/002.create-table.app_roles.sql
-- v0.8.4

-- \echo 'If necessary, perform -- DROP TABLE IF EXISTS app_roles;'

-- \echo '1. Creating the app_roles table'

CREATE TABLE app_roles (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , name           varchar (100) NOT NULL DEFAULT 'Role name'
    , level          smallint NOT NULL DEFAULT 1000
    , description    varchar (200) NULL DEFAULT ''
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_app_roles_guid PRIMARY KEY (guid)
    );

CREATE unique index idx_app_roles_uniq_id
	ON app_roles (id);


INSERT INTO app_roles 
   ( guid, id, name, level, description )
VALUES
   ('71eea083-d818-4557-89fe-29eb950881aa', 1, 'PRODUCT_INSTANCE_OWNER', 0, 'Product instance owner'),
   ('71eea083-d818-4557-89fe-29eb950881ac', 2, 'EDITOR', 2, 'Has the right to edit content'),
   ('71eea083-d818-4557-89fe-29eb950881ad', 3, 'READER', 7, 'Has the right to read content'),
   ('71eea083-d818-4557-89fe-29eb950881ab', 4, 'ANONYMOUS', 1000, 'Non-registered user having access to instance');


-- \echo 'List columns of the created table app_roles'
SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid='public.app_roles'::regclass
   AND    attnum>0
   AND    NOT attisdropped
   ORDER  BY attnum ; 

-- \echo 'Update time on every EXECUTE trigger:'
CREATE TRIGGER trg_set_update_time_on_app_roles
	BEFORE UPDATE ON app_roles 
	FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
	FROM pg_trigger
	WHERE NOT tgisinternal
	AND tgrelid='app_roles'::regclass;<file_sep>-- file: src/sql/pgsql/scripts/admin/list-schema-tables.sql
-- usage: 
/**
alias psql="PGPASSWORD=${postgres_sys_usr_admin_pw:-} psql -v -t -X -w -U ${postgres_sys_usr_admin:-}"
psql -d dev_qto < src/sql/pgsql/scripts/admin/list-schema-tables.sql 
*/

      INSERT INTO app_items ( id,name,is_menu)
      SELECT  
           ROW_NUMBER () OVER (ORDER BY table_name) as id
         , table_name as name
         , '1'
      FROM information_schema.tables  
      WHERE 1=1 
      AND table_type = 'BASE TABLE' 
      AND table_schema = 'public' 
      ORDER BY table_name
;

/*
      SELECT  
           ROW_NUMBER () OVER (ORDER BY table_name) as row_id
         , table_name as name
      FROM information_schema.tables  
      WHERE 1=1 
      AND table_type = 'BASE TABLE' 
      AND table_schema = 'public' 
      ORDER BY table_name
;
*/
-- eof file: src/sql/pgsql/scripts/admin/list-schema-tables.sql
<file_sep>-- \echo 'If necessary, perform -- DROP TABLE IF EXISTS app_routes;'

-- \echo '6. Creating the app_routes table'

CREATE TABLE app_routes (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , prio           smallint NOT NULL DEFAULT 1
    , is_open        bool NOT NULL default true
    , is_open_in     bool NOT NULL default false
    , has_subject    bool NOT NULL default true
    , is_backend     bool NOT NULL default true
    , name           varchar (200) NOT NULL DEFAULT 'Route name'
    , description    varchar (4000) DEFAULT ''
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_app_routes_guid PRIMARY KEY (guid)
    );

CREATE UNIQUE INDEX idx_uniq_app_routes_id
	ON app_routes (id);

-- \echo 'List columns of the created table app_routes'
SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid='public.app_routes'::regclass
   AND    attnum>0
   AND    NOT attisdropped
   ORDER  BY attnum; 

-- \echo 'Update time on every EXECUTE trigger:'
CREATE TRIGGER trg_set_update_time_on_app_routes
	BEFORE UPDATE ON app_routes
	FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
	FROM pg_trigger
	WHERE NOT tgisinternal
	AND tgrelid='app_routes'::regclass;


--
-- TOC entry 3313 (class 0 OID 60445)
-- Dependencies: 270
-- Data for Name: app_routes; Type: TABLE DATA; Schema: public; Owner: usrdevqtoadmin
--

COPY public.app_routes (guid, id, prio, is_open, is_open_in, has_subject, is_backend, name, description, update_time) FROM stdin;
9b2bfd99-bedc-4c8d-8a89-1051a57386e7	200228153152	1	f	f	t	t	insert	The insert route tells the application layer to insert a row via post 	2020-05-13 17:28:08
e86764c7-c25b-4391-a24b-d155f3b7032a	200228080831	1	f	f	t	t	truncate	The TRUNCATE route tells the qto Application Layer to truncate a whole item table 	2020-05-13 17:28:08
4de2af7f-d23e-4f3c-a3e1-b84101cbf506	200228153439	1	f	f	t	t	hidelete	The deletion of hierarchical row in the db	2020-05-13 17:28:08
3f651cb4-0702-4dc6-8368-a1c7d1e45ba0	200228080425	1	f	f	t	t	delete	the DELETE route tells the qto Application Layer to delete a row in an item table from DELETE request provided json data	2020-05-13 17:28:08
03432642-6bcd-4bbc-bc39-674062e4403f	200228080115	1	f	f	t	t	select	The SELECT route tells the qto Application Layer  to select the data of an item in json data format ( aka qto api compliant table )	2020-05-13 17:28:08
8799afd6-5852-4e6d-a5e9-e8f8b07e23d4	200228153347	1	f	f	t	t	hicreate	The creation of a hierarchical row in the db	2020-05-13 17:28:08
b2a70ed6-e7d0-42ce-8e61-9138031a0e23	200228080331	1	f	f	t	t	update	The UPDATE route tells the qto Application Layer to update a single value in a row of an item table from POST request provided json data	2020-05-13 17:28:08
e0f2627a-e395-4f4a-a01a-793a2dde8bc8	200228153603	1	f	f	t	f	view	The viewing of a hierarchichal item in the view doc UI 	2020-05-13 17:28:08
ff5202c2-aebf-4007-af6d-e287addb8f93	200228153538	1	f	f	t	t	export	The export of data for an item , for example export as xls	2020-05-13 17:28:08
891ef0ee-bbf5-41c9-82d4-cfbb5982a59a	200228080240	1	f	f	t	t	create	The CREATE route tells the qto Application Layer  to create a row in an item table from POST request with json data	2020-05-13 17:28:08
05a29477-e337-4f63-8003-38a0f0ba4c01	200403234321	1	f	f	t	t	call-func	the back-end call of the report route°	2020-05-13 17:28:08
e8b82d1b-fbd9-4a55-9119-2e757ec48bb9	200228153215	1	f	f	t	t	hiselect	The hiselect selects a hierarchical data from a hierarchichal table	2020-05-13 17:28:08
8b4aed40-bd21-431e-bc5c-a491177e05fb	200228153323	1	t	t	f	f	login	The login route provides the login functionality	2020-05-13 19:18:01
77096ddd-4eb4-4dfb-ac13-2ad20a041e19	200404002417	1	t	t	f	t	select tables	lists all the tables in the project database	2020-05-13 19:15:55
e98492e3-73f4-4d82-86c6-1abc8174a0ac	200228080729	1	f	t	f	t	query	The QUERY route tells the qto Application Layer  to display the search results in json data format, produced by querying all the item tables in the db	2020-05-13 19:16:23
6dc3375f-781b-4d4f-8196-f18e1a75cc5c	200228153308	1	t	t	f	f	logon	The Logon routes logs on a user	2020-05-13 19:18:15
1ec738f2-865c-433d-b5a0-6f5e67882659	200228080606	1	f	t	f	f	search	The SEARCH route tells the qto Application Layer  to display the search results in the qto UI, produced by querying all the item tables in the db	2020-05-13 19:15:58
0f34e0b1-743c-4ad5-a391-f8f7a119a9b6	200403233318	1	t	t	t	t	serve	The routes serving errors and warnings from redirects such as 403 forbidden	2020-05-13 19:15:26
e7506dcd-3c2d-4b44-a126-45cd47deb8a0	200502002643	1	f	f	t	t	select-my	select only the records belonging to the logged in user	2020-05-13 19:15:34
cc2ae685-01f8-4375-ae15-cf48b9b54e49	200228153513	1	f	t	t	t	select-item-meta-for	The retrieval of the meta-data for an item	2020-05-13 19:15:38
81167627-8853-4ca1-ae9e-171890e80098	200309203853	1	f	f	t	f	report	The route used for calling reports ( which are postgres result set returning functions)	2020-05-13 19:16:02
9b6dbcd5-da0d-4d36-be4c-e1944dd03d94	200403234539	1	t	t	f	t	select-databases	list the databases in the configured postgres server to the AL	2020-05-13 19:15:45
07c0cbb7-07e4-477e-8ec3-d80e76a2ab3c	200502002514	1	f	f	t	f	list-my	list only the records belonging to the logged in user	2020-05-13 19:16:39
15b6523e-8e2c-45bf-bcc4-13bc8da1bf86	200228055434	1	f	f	t	f	list	The LIST route tells the qto Application Layer  to list in the qto UI the content of an item ( aka qto api compliant table )	2020-05-13 19:17:13
1854a713-d29e-4abd-9c29-0b27d08ebea2	200513223249	1	f	f	t	t	select-col	selects ONLY a single column from a table ...	2020-05-13 19:33:40
\.
<file_sep>#!/usr/bin/env bash

usage(){
   echo provide the PROJ_CONF_FILE as the first cmd arg - \$1
   echo source $0 ~/opt/csitea/csitea.0.8.2.prd.ubuntu@csitea.net/cnf/env/prd.env.json
}

do_set_vars(){
   set -eu -o pipefail
   shopt -s expand_aliases  # src: http://chiefsandendians.blogspot.com/2010/07/linux-scripts-and-alias.html
   original_dir=$(perl -e 'use File::Basename; use Cwd "abs_path"; print dirname(abs_path(@ARGV[0]));' -- "$0")
   export product_dir=$(cd $original_dir; echo `pwd`)
   cd $product_dir
}

main(){
   do_set_vars
   test -z ${1:-} && usage
   test -z ${1:-} || {
      export PROJ_CONF_FILE=$1
      export PROJ_INSTANCE_DIR=$(cd `dirname $PROJ_CONF_FILE`/../..; echo `pwd`)
      source $product_dir/lib/bash/funcs/export-json-section-vars.sh
      do_export_json_section_vars $PROJ_CONF_FILE '.env.db';
	  do_flush_screen
      sleep 1
	  
      alias psql="PGPASSWORD=${postgres_sys_usr_admin_pw:-} psql -v -t -X -w -U ${postgres_sys_usr_admin:-} --port $postgres_rdbms_port --host $postgres_rdbms_host"
      alias|sort ; sleep 1 ;
	  do_flush_screen
   }
}

main "$@" #Action !!!
<file_sep>#!/usr/bin/env bash

original_dir=$(perl -e 'use File::Basename; use Cwd "abs_path"; print dirname(abs_path(@ARGV[0]));' -- "$0")

# Add the PostgreSQL PGP key to verify their Debian packages.
# It should be the same key as https://www.postgresql.org/media/keys/ACCC4CF8.asc
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -

# add the PostgreSQL's repository
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main"' \
   > /etc/apt/sources.list.d/pgdg.list

# add the most basic binaries
apt-get clean && apt-get update && apt-get install -f -y postgresql-server-dev-11 postgresql-client-11 postgresql-contrib-11

## ensure the postresql starts on boot 
sudo update-rc.d postgresql enable

mkdir -p /etc/postgresql/11/main/
mkdir -p /var/lib/postgresql/11/main

echo "postgres:postgres" | chpasswd
echo 'export PS1="`date "+%F %T"` \u@\h  \w \\n\\n  "' >> /var/lib/postgresql/.bashrc


## add the uuid generation capability enabling extensions
/etc/init.d/postgresql restart
sudo -u postgres psql -c "CREATE USER usrqtoadmin WITH SUPERUSER CREATEROLE CREATEDB REPLICATION BYPASSRLS PASSWORD '<PASSWORD>';"
sudo -u postgres psql -c "grant all privileges on database postgres to usrqtoadmin ;"
sudo -u postgres createdb -O postgres postgres
sudo -u postgres psql template1 -c 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp";'
sudo -u postgres psql template1 -c 'CREATE EXTENSION IF NOT EXISTS "pgcrypto";'
sudo -u postgres psql template1 -c 'CREATE EXTENSION IF NOT EXISTS "dblink";' 

psql_cnf_dir='/etc/postgresql/11/main'
test -f $psql_cnf_dir/pg_hba.conf && \
   sudo cp -v $psql_cnf_dir/pg_hba.conf $psql_cnf_dir/pg_hba.conf.orig.bak && \
   sudo cp -v $PRODUCT_DIR/cnf/postgres/$psql_cnf_dir/pg_hba.conf $psql_cnf_dir/pg_hba.conf && \
   sudo chown postgres:postres $psql_cnf_dir

test -f $psql_cnf_dir/postgresql.conf && \
   sudo cp -v $psql_cnf_dir/postgresql.conf $psql_cnf_dir/postgresql.conf.orig && \
   sudo cp -v $PRODUCT_DIR/cnf/postgres/$psql_cnf_dir/postgresql.conf $psql_cnf_dir/postgresql.conf

chown -R postgres:postgres "/etc/postgresql" && \
    chown -R postgres:postgres "/var/lib/postgresql" && \
    chown -R postgres:postgres "/etc/postgresql/11/main/pg_hba.conf" && \
    chown -R postgres:postgres "/etc/postgresql/11/main/postgresql.conf"
<file_sep>doPublishDbdumpToS3(){
   test -z "${PROJ_INSTANCE_DIR-}" && PROJ_INSTANCE_DIR="$PRODUCT_DIR"
   #
   do_export_json_section_vars $PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json '.env.db'
   do_export_json_section_vars $PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json '.env.aws'

   dump_file="$(find $PROJ_INSTANCE_DIR/dat/mix/ -type f -name '*.insrts.dmp.sql'|sort -nr|head -n 1)"
   dump_file_name=$(basename $dump_file)

   export AWS_DEFAULT_PROFILE='default'
   aws configure list
   # remove the grants to disable full publicity of the data ...
   set +x
   aws s3 --profile default cp "$dump_file" "s3://$bucket/$postgres_app_db"'.latest.insrts.dmp.sql'
   set -x

   cat << EOF
				 IF error occurs check that your aws credentials file ~/.aws/credentials looks as follws
				 sudo ntpdate ntp.ubuntu.com
				 cat ~/.aws/credentials
				 [default]
				 aws_access_key_id = <KEY>
				 aws_secret_access_key = <KEY>
EOF
}
<file_sep>
	EXECUTE format( 'REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM %I'
			, current_setting('myvars.postgres_app_usr', true)::text 
			, current_setting('myvars.postgres_app_usr_pw', true)::text
	);
	EXECUTE format( 'REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM %I'
			, current_setting('myvars.postgres_app_usr', true)::text 
			, current_setting('myvars.postgres_app_usr_pw', true)::text
	);
	EXECUTE format( 'REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM %I'
			, current_setting('myvars.postgres_app_usr', true)::text 
			, current_setting('myvars.postgres_app_usr_pw', true)::text
	);
	EXECUTE format( 'REVOKE ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA public FROM %I'
			, current_setting('myvars.postgres_app_usr', true)::text 
			, current_setting('myvars.postgres_app_usr_pw', true)::text
	);
	EXECUTE format( 'REASSIGN OWNED BY %I TO postgres;'
			, current_setting('myvars.postgres_app_usr', true)::text 
	);
	EXECUTE format( 'DROP OWNED BY %I CASCADE' 
			, current_setting('myvars.postgres_app_usr', true)::text 
	);
<file_sep>#!/bin/bash

do_delete_aws_vpc(){


   test -z ${PROJ_CONF_FILE:-} && export PROJ_CONF_FILE="$PROJ_INSTANCE_DIR/cnf/env/$ENV.env.json"

   do_export_json_section_vars $PROJ_CONF_FILE '.env.aws'
   export AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION

	# probably a buggy one but just to get you start with something
	# ensure your default output is json + you have default region ...
	aws ec2 describe-internet-gateways --filters 'Name=attachment.vpc-id,Values='$vpc_id \
	  | jq -r ".InternetGateways[].InternetGatewayId"
  # terminate all vpc instances
  while read -r instance_id ; do
	  aws ec2 terminate-instances --instance-ids $instance_id
  done < <(aws ec2 describe-instances --filters 'Name=vpc-id,Values='$vpc_id \
	  | jq -r '.Reservations[].Instances[].InstanceId')

  while read -r sg ; do
	  aws ec2 delete-security-group --group-id $sg
  done < <(aws ec2 describe-security-groups --filters 'Name=vpc-id,Values='$vpc_id \
	  | jq -r '.SecurityGroups[].GroupId')

  # delete all vpc subnets
  while read -r subnet_id ; do
	  aws ec2 delete-subnet --subnet-id "$subnet_id"
  done < <(aws ec2 describe-subnets --filters 'Name=vpc-id,Values='$vpc_id | jq -r '.Subnets[].SubnetId')

  # delete all vpc route tables
  while read -r rt_id ; do
	  aws ec2 delete-route-table --route-table-id $rt_id ;
  done < <(aws ec2 describe-route-tables --filters 'Name=vpc-id,Values='$vpc_id | \
	  jq -r .RouteTables[].RouteTableId)

  # detach first all vpc internet gateways
  while read -r ig_id ; do
	  aws ec2 detach-internet-gateway --internet-gateway-id $ig_id --vpc-id $vpc_id
  done < <(aws ec2 describe-internet-gateways --filters 'Name=attachment.vpc-id,Values='$vpc_id  \
	  | jq -r ".InternetGateways[].InternetGatewayId")

  # delete than all vpc internet gateways
  while read -r ig_id ; do
	  aws ec2 delete-internet-gateway --internet-gateway-id $ig_id --vpc-id $vpc_id
  done < <(aws ec2 describe-internet-gateways --filters 'Name=attachment.vpc-id,Values='$vpc_id  \
	  | jq -r ".InternetGateways[].InternetGatewayId")

  # attempt to the delete the whole vpc
  aws ec2 delete-vpc --vpc-id=$vpc_id

   aws ec2 describe-internet-gateways --filters 'Name=attachment.vpc-id,Values='$vpc | grep InternetGatewayId
   aws ec2 describe-subnets --filters 'Name=vpc-id,Values='$vpc | grep SubnetId
   aws ec2 describe-route-tables --filters 'Name=vpc-id,Values='$vpc | grep RouteTableId
   aws ec2 describe-network-acls --filters 'Name=vpc-id,Values='$vpc | grep NetworkAclId
   aws ec2 describe-vpc-peering-connections --filters 'Name=requester-vpc-info.vpc-id,Values='$vpc | grep VpcPeeringConnectionId
   aws ec2 describe-vpc-endpoints --filters 'Name=vpc-id,Values='$vpc | grep VpcEndpointId
   aws ec2 describe-nat-gateways --filter 'Name=vpc-id,Values='$vpc | grep NatGatewayId
   aws ec2 describe-security-groups --filters 'Name=vpc-id,Values='$vpc | grep GroupId
   aws ec2 describe-instances --filters 'Name=vpc-id,Values='$vpc | grep InstanceId
   aws ec2 describe-vpn-connections --filters 'Name=vpc-id,Values='$vpc | grep VpnConnectionId
   aws ec2 describe-vpn-gateways --filters 'Name=attachment.vpc-id,Values='$vpc | grep VpnGatewayId
   aws ec2 describe-network-interfaces --filters 'Name=vpc-id,Values='$vpc | grep NetworkInterfaceId

#   while read -r key_pair_id; do
#      aws echo ec2 delete-key-pair --key-name $key_pair_id ;
#   done < <(aws ec2 describe-key-pairs | jq -r '.[] | .[] | .KeyPairId')
}
<file_sep># todo




# eof file: src/make/local-setup-tasks.incl.mk
<file_sep># src/bash/qto/funcs/run-data-load-tests.func.sh

# v1.0.9
# ---------------------------------------------------------
# cat doc/txt/qto/funcs/run-data-load-tests.func.txt
# ---------------------------------------------------------
doRunDataLoadTests(){

	do_log "DEBUG START doRunDataLoadTests"
	
   do_log "INFO START test the Select Controller "
   do_log " $postgres_app_db/select-tables"
   do_log " $postgres_app_db/select/<<table-name>>"
   test_script='src/perl/qto/t/lib/Qto/App/benchmarks/LoadTestSelect.t'
   do_log "INFO START $test_script"
   perl src/perl/qto/t/lib/Qto/Controller/LoadTestSelect.pl
   test $? -ne 0 && return
   do_log "INFO STOP  $test_script"
	echo -e "\n\n\n" 

	do_log "DEBUG STOP  doRunDataLoadTests"
}
# eof func doRunDataLoadTests


# eof file: src/bash/qto/funcs/run-data-load-tests.func.sh
<file_sep># src/bash/qto/funcs/db-to-gsheet.func.sh

# v1.0.9
# ---------------------------------------------------------
# cat doc/txt/qto/funcs/db-to-gsheet.func.txt
# ---------------------------------------------------------
doDbToGsheet(){

	do_log "DEBUG START doDbToGsheet"
	
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!
   do_log "INFO START testing db-to-gsheet"
   test -z ${tables+x} && export tables='daily_issues'
   perl src/perl/qto/script/qto.pl --do db-to-gsheet --tables $tables
   exit_code=$?
   do_log "INFO doRunQto exit_code $exit_code"
   test $exit_code -ne 0 && do_exit $exit_code "failed to run qto.pl"  
   



	do_log "DEBUG STOP  doDbToGsheet"
}
# eof func doDbToGsheet


# eof file: src/bash/qto/funcs/db-to-gsheet.func.sh
<file_sep># src/bash/qto/funcs/remove-package.func.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doRemovePackage comments ...
# ---------------------------------------------------------
doRemovePackage(){

	do_log "DEBUG START doRemovePackage"
	
	cat doc/txt/qto/tmpl/remove-package.func.txt
	sleep 2
	# add your action implementation code here ... 

	do_log "DEBUG STOP  doRemovePackage"
}
# eof func doRemovePackage


# eof file: src/bash/qto/funcs/remove-package.func.sh
<file_sep>/**
 * usage:
   cd src/js/node/pp-ui-tests/ ; npm install puppeteer; cd -
   cd src/js/node/pp-ui-tests/ ; npm run ; cd -
   cd src/js/node/pp-ui-tests/ ; node test-open-login.js; cd -
   cd src/js/node/pp-ui-tests/ ; npm test; cd -
 * @name open the login page
 *
 * @desc Gets the value of commonly used HTML form elements using page.$eval()
 * chk also:
 * src/js/node/pp-ui-tests/package.json
 *
 */
const puppeteer = require('puppeteer');
const { expect } = require('chai');
const _ = require('lodash');
const globalVariables = _.pick(global, ['browser', 'expect']);

// puppeteer options
const opts = {
  headless: false,
  slowMo: 100,
  timeout: 10000
};

// expose variables
before (async function () {
  global.expect = expect;
  global.browser = await puppeteer.launch(opts);
});

// close browser and reset global variables
after (function () {
  browser.close();

  global.browser = globalVariables.browser;
  global.expect = globalVariables.expect;
});

describe('sample test', function () {
  it('should work', async function () {
    console.log(await browser.version());

    expect(true).to.be.true;
  });


});
<file_sep># usage example
# install-the-bot: demand_var-ENV demand_var-TGT_ORG do-build-$(component)-docker-img
demand_var-%:
	@if [ "${${*}}" = "" ]; then \
		echo "the var \"$*\" is not set, do set it by: export $*='value'"; \
		exit 1; \
	fi

# function usage example
# install-the-bot:
# 	$(call demand-var,ENV)
# 	$(call demand-var,TGT_ORG)
# 	do-build-$(component)-docker-img
define demand-var
	@if [ "${${1}}" = "" ]; then \
		echo "the var \"$1\" is not set, do set it by: export $1='value'"; \
		exit 1; \
	fi
endef
<file_sep># src/bash/qto/funcs/increase-date.test.sh

doTestIncreaseDate(){
   set -eu
	sleep "$sleep_interval"
   bash src/bash/qto/qto.sh -a increase-date
   test $exit_code -ne 0 && return
}


<file_sep>doScrambleConfs(){
   eval $(perl -I$HOME/perl5/lib/perl5 -Mlocal::lib)

   while read -r file ; do
      cp -v $file $file.bak
		script=$(cat <<'EOF'
			use strict; use warnings; binmode STDOUT, ":utf8"; use utf8; use JSON; use Data::Printer;
			sub rndStr{ join'', @_[ map{ rand @_ } 1 .. shift ] }
			my $sjson;
			{
			  local $/; #Enable 'slurp' mode
			  open my $fh, "<", $ARGV[0];
			  $sjson = <$fh>;
			  close $fh;
			}
			my $data = decode_json($sjson);
			#p $data ; 
			# basically scramble the passwords
			$data->{'env'}->{'aws'}->{'AWS_ACCESS_KEY_ID'} = rndStr 12, 'A'..'Z', 0..9, 'a'..'z' ; 
			$data->{'env'}->{'aws'}->{'AWS_SECRET_ACCESS_KEY'} = rndStr 12, 'A'..'Z', 0..9, 'a'..'z' ; 
			$data->{'env'}->{'db'}->{'postgres_usr_pw'} = rndStr 12, 'A'..'Z', 0..9, 'a'..'z' ; 
			$data->{'env'}->{'db'}->{'postgres_rdbms_usr_pw'} = rndStr 12, 'A'..'Z', 0..9, 'a'..'z' ; 
			$data->{'env'}->{'db'}->{'postgres_app_usr_pw'} = rndStr 12, 'A'..'Z', 0..9, 'a'..'z' ; 
			$data->{'env'}->{'db'}->{'postgres_sys_usr_admin_pw'} = rndStr 12, 'A'..'Z', 0..9, 'a'..'z' ; 
			$data->{'env'}->{'db'}->{'app_user_pwd'} = rndStr 12, 'A'..'Z', 0..9, 'a'..'z' ; 
			$data->{'env'}->{'db'}->{'AdminEmail'} = rndStr 12, 'A'..'Z', 0..9, 'a'..'z' ; 
         my $json = JSON->new->allow_nonref;
			open my $fh, ">", $ARGV[0];
			print $fh $json->pretty->encode($data);
			close $fh;
EOF
)
		perl -e "$script" $file
      git add $file
	done < <(find cnf/env/ -type f| grep -v '.bak')

}
<file_sep>do_configure_aws_keys(){

	# set up confs
   source $PRODUCT_DIR/.env ; ENV=$ENV
   test -z ${PROJ_CONF_FILE:-} && export PROJ_CONF_FILE="$PRODUCT_DIR/cnf/env/$ENV.env.json"
   source $PRODUCT_DIR/lib/bash/funcs/export-json-section-vars.sh
   do_export_json_section_vars $PROJ_CONF_FILE '.env.aws'

   mkdir -p ~/.aws/

   cat << EOF_AWS | tee ~/.aws/credentials
[default]
aws_access_key_id = $AWS_ACCESS_KEY_ID
aws_secret_access_key = $AWS_SECRET_ACCESS_KEY
EOF_AWS
}
<file_sep># src/make/install-dockers.func.mk
# only the install dockers calls here ...
# TODO: figure a more elegant and generic way to avoid this copy paste ...
#

SHELL = bash
PRODUCT := $(shell basename $$PWD)
TGT_ORG := $(shell export TGT_ORG=$${TGT_ORG:-csitea}; echo $${TGT_ORG})


.PHONY: install-devops  ## @-> setup the whole local devops environment
install-devops:
	$(call install-img,devops,3000)
<file_sep># file: src/bash/qto/funcs/mojo-hypnotoad-stop.func.sh
#
# ---------------------------------------------------------
# cat doc/txt/qto/funcs/mojo-hypnotoad-stop.func.txt
# ---------------------------------------------------------
doMojoHypnotoadStop(){

   do_export_json_section_vars $PRODUCT_DIR/cnf/env/$ENV.env.json '.env.app'
   test -z "${mojo_hypnotoad_port:-}" && export mojo_hypnotoad_port=8080

   # try nicely first to stop the hypnotoad worker processes
   while read -r child_of_1_pid; do 
      echo the child_of_1_pid : $child_of_1_pid
      while read -r listening_on_port_pid; do 
         echo and the listening_on_port_pid: $listening_on_port_pid
         while read -r pid_to_stop; do 
            echo trying first to stop gracefully the following pid_to_sop  $pid_to_stop; 
            sudo kill -3 $pid_to_stop # try nicely first 
         done < <(ps -ef | grep $child_of_1_pid|grep $RUN_UNIT |grep -v grep|awk '{print $2}'); 
       done < <(lsof -i:${mojo_hypnotoad_port:-} -t|grep $child_of_1_pid)
   done < <(pgrep -P 1)
   
  # no more nice guy .. - kill those which did not stop ...
   while read -r child_of_1_pid; do 
      while read -r listening_on_port_pid; do 
         while read -r pid_to_kill; do 
            echo trying then to kill the following pid_to_kill $pid_to_kill; 
            sudo kill -9 $pid_to_kill # try forcefully then ...
         done < <(ps -ef | grep $child_of_1_pid|grep $RUN_UNIT |grep -v grep|awk '{print $2}'); 
       done < <(lsof -i:${mojo_hypnotoad_port:-} -t|grep $child_of_1_pid)
   done < <(pgrep -P 1)

   sudo service nginx stop
}

<file_sep>#!/usr/bin/env bash

curr_branch=$(git rev-parse --abbrev-ref HEAD)

test "$curr_branch" = 'dev' && echo 'ARE YOU SURE !!! PUSHING TO dev !!!!'
test "$curr_branch" = 'dev' && sleep 5
test "$curr_branch" = 'tst' && echo 'ARE YOU SURE !!! PUSHING TO tst !!!!'
test "$curr_branch" = 'tst' && sleep 5
test "$curr_branch" = 'prd' && echo 'ARE YOU SURE !!! PUSHING TO prd !!!!'
test "$curr_branch" = 'prd' && sleep 5
test "$curr_branch" = 'master' && echo 'ARE YOU SURE !!! PUSHING TO master !!!!'
test "$curr_branch" = 'master' && sleep 5

echo "checking for own todo left-overs ... ""$(grep --exclude-dir '.git' -ri 'todo:ysg' . | grep -v 'pre-commit' | wc -l)"" found."
grep --exclude-dir '.git' -rnHi 'todo:ysg' . | grep -v 'pre-commit'

sleep 2
exit 0
# exit $(grep -ri 'todo:ysg' . | grep -v 'pre-commit' | wc -l)
<file_sep># src/bash/qto/funcs/remove-package-files.test.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doTestRemovePackageFiles comments ...
# ---------------------------------------------------------
doTestRemovePackageFiles(){

	do_log "DEBUG START doTestRemovePackageFiles"
	
	cat doc/txt/qto/tests/remove-package-files.test.txt
	
	test -z "$sleep_interval" || sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!

	do_log "DEBUG STOP  doTestRemovePackageFiles"
}
# eof func doTestRemovePackageFiles


# eof file: src/bash/qto/funcs/remove-package-files.test.sh
<file_sep>---- DROP TABLE IF EXISTS test_update_table CASCADE ; 

SELECT 'create the "test_update_table" table'
;

-- src: http://mikehillyer.com/articles/managing-hierarchical-data-in-mysql/
   CREATE TABLE test_update_table (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , seq            integer NULL
    , name           varchar (200) NOT NULL DEFAULT 'name...'
    , password       varchar (200) NULL
    , description    varchar (4000)
    , CONSTRAINT pk_test_update_table_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );


SELECT 'Display the columns of the just created table'
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.test_update_table'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 
<file_sep>-- START create table test_hierarchy_doc
-- DROP TABLE IF EXISTS test_hierarchy_doc ; 

SELECT 'create the "test_hierarchy_doc" table' as "---"
;

-- src: http://mikehillyer.com/articles/managing-hierarchical-data-in-mysql/
   CREATE TABLE test_hierarchy_doc (
      guid           UUID NOT NULL DEFAULT gen_rANDom_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , level          integer NULL
    , seq            integer NULL
    , lft            integer NULL 
    , rgt            integer NULL 
    , name           varchar (200) NOT NULL DEFAULT 'name...'
    , description    varchar (4000) NULL
    , src            varchar (4000) NULL
    , formats        text NULL
    , CONSTRAINT pk_test_hierarchy_doc_guid PRIMARY KEY (guid)
    ) WITH (
      OIDS=FALSE
    );

-- STOP  create table test_hierarchy_doc
-- --------------------------------------------------------



SELECT 'Display the columns of the just created table' as "---"
; 

   SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid = 'public.test_hierarchy_doc'::regclass
   AND    attnum > 0
   AND    NOT attisdropped
   ORDER  BY attnum
   ; 


 
 TRUNCATE TABLE test_hierarchy_doc ; 
 INSERT into test_hierarchy_doc ( id,level,seq,lft,rgt,name ) values ( 0 ,0,1 ,1,23,'name-01 0.0.0');
 INSERT into test_hierarchy_doc ( id,level,seq,lft,rgt,name ) values ( 1 ,1,2 ,2,9,'name-02 1.0.0');
 INSERT into test_hierarchy_doc ( id,level,seq,lft,rgt,name ) values ( 2 ,2,3 ,3,8,'name-03 1.1.0');
 INSERT into test_hierarchy_doc ( id,level,seq,lft,rgt,name ) values ( 3 ,3,4 ,4,5,'name-04 1.1.1');
 INSERT into test_hierarchy_doc ( id,level,seq,lft,rgt,name ) values ( 4 ,3,5 ,6,7,'name-05 1.1.2');
 INSERT into test_hierarchy_doc ( id,level,seq,lft,rgt,name ) values ( 5 ,1,6 ,10,13,'name-06 2.0.0');
 INSERT into test_hierarchy_doc ( id,level,seq,lft,rgt,name ) values ( 6 ,2,7 ,11,12,'name-07 2.1.0');
 INSERT into test_hierarchy_doc ( id,level,seq,lft,rgt,name ) values ( 7 ,1,8 ,14,23,'name-08 3.0.0');
 INSERT into test_hierarchy_doc ( id,level,seq,lft,rgt,name ) values ( 8 ,2,9 ,15,16,'name-09 3.1.0');
 INSERT into test_hierarchy_doc ( id,level,seq,lft,rgt,name ) values ( 9 ,2,10,17,22,'name-10 3.2.0');
 INSERT into test_hierarchy_doc ( id,level,seq,lft,rgt,name ) values ( 10,3,11,18,21,'name-11 3.2.1');
 INSERT into test_hierarchy_doc ( id,level,seq,lft,rgt,name ) values ( 11,4,12,19,20,'name-12 3.2.1.1');
 SELECT * FROM test_hierarchy_doc ORDER BY seq ; 

SELECT 'retrieve the full branch from the parent id with sequence eq to 1' as "---" ;
SELECT * FROM (
	SELECT node.* FROM test_hierarchy_doc AS node, test_hierarchy_doc AS parent
	WHERE 1=1 AND node.lft
	BETWEEN parent.lft AND parent.rgt
	AND parent.seq = 1)  AS dyn_sql
WHERE 1=1
ORDER BY seq
;
SELECT 'retrieve the full branch from the parent id with sequence eq to 2' as "---" ;
SELECT * FROM (
	SELECT node.* FROM test_hierarchy_doc AS node, test_hierarchy_doc AS parent
	WHERE 1=1 AND node.lft
	BETWEEN parent.lft AND parent.rgt
	AND parent.seq = 2)  AS dyn_sql
WHERE 1=1
ORDER BY seq
;
SELECT 'retrieve the full branch from the parent id with sequence eq to 6' as "---" ;
SELECT * FROM (
	SELECT node.* FROM test_hierarchy_doc AS node, test_hierarchy_doc AS parent
	WHERE 1=1 AND node.lft
	BETWEEN parent.lft AND parent.rgt
	AND parent.seq = 6)  AS dyn_sql
WHERE 1=1
ORDER BY seq
;
SELECT 'retrieve the full branch from the parent id with sequence eq to 8' as "---" ;
SELECT * FROM (
	SELECT node.* FROM test_hierarchy_doc AS node, test_hierarchy_doc AS parent
	WHERE 1=1 AND node.lft
	BETWEEN parent.lft AND parent.rgt
	AND parent.seq = 8)  AS dyn_sql
WHERE 1=1
ORDER BY seq
;


<file_sep># src/bash/qto/funcs/clone-project.test.sh

# v1.0.9
# ---------------------------------------------------------
# todo: add doTestCloneProject comments ...
# ---------------------------------------------------------
doTestCloneProject(){

	do_log "DEBUG START doTestCloneProject"
	
	cat doc/txt/qto/tests/clone-project.test.txt
	
	sleep "$sleep_interval"
	# add your action implementation code here ... 
	# Action !!!

	do_log "DEBUG STOP  doTestCloneProject"
}
# eof func doTestCloneProject


# eof file: src/bash/qto/funcs/clone-project.test.sh
<file_sep># src/bash/qto/funcs/run-functional-tests.test.sh

# v1.2.9
# ---------------------------------------------------------
# cat doc/txt/qto/tests/perl/run-functional-tests.test.txt
# ---------------------------------------------------------
doTestRunFunctionalTests(){

	do_log "DEBUG START doTestRunFunctionalTests"
	
	sleep "$sleep_interval"
   # Action !!!	
	bash src/bash/qto/qto.sh -a run-functional-tests
	
	do_log "DEBUG STOP  doTestRunFunctionalTests"
}
# eof func doTestRunFunctionalTests


# eof file: src/bash/qto/funcs/run-functional-tests.test.sh
<file_sep>CREATE TABLE app_instance_releases (
      guid           UUID NOT NULL DEFAULT gen_random_uuid()
    , id             bigint UNIQUE NOT NULL DEFAULT cast (to_char(current_timestamp, 'YYMMDDHH12MISS') as bigint) 
    , app_ver        varchar (10) NOT NULL DEFAULT '0.0.0'
    , app_git_ref    varchar (9) NOT NULL DEFAULT ''
    , db_ver         varchar (10) NOT NULL DEFAULT '0.0.0'
    , db_git_ref     varchar (9) NOT NULL DEFAULT ''
    , env            varchar (3) NOT NULL DEFAULT 'dev'
    , proj_db_name   varchar (20) NOT NULL DEFAULT 'qto'
    , dns_name       varchar (20) NOT NULL DEFAULT 'qto.fi'
    , name           varchar (200) NOT NULL DEFAULT 'release title ...'
    , description    varchar (4000) NOT NULL DEFAULT 'release description add links to issues etc.' 
    , update_time    timestamp DEFAULT DATE_TRUNC('second', NOW())
    , CONSTRAINT pk_app_instance_releases_guid PRIMARY KEY (guid)
    );


CREATE UNIQUE INDEX idx_uniq_app_instance_releases_id
	ON app_instance_releases (id);

-- \echo 'List columns of the created table app_instance_releases'

SELECT attrelid::regclass, attnum, attname
   FROM   pg_attribute
   WHERE  attrelid='public.app_instance_releases'::regclass
   AND    attnum>0
   AND    NOT attisdropped
   ORDER  BY attnum; 

-- \echo 'Update time on every EXECUTE trigger:'
CREATE TRIGGER trg_set_update_time_on_app_instance_releases
	BEFORE UPDATE ON app_instance_releases
	FOR EACH ROW EXECUTE PROCEDURE fnc_set_update_time();

SELECT tgname
	FROM pg_trigger
	WHERE NOT tgisinternal
	AND tgrelid='app_instance_releases'::regclass;
