Add app.py and the screencoder repo

screencoder/.gitignore

@@ -0,0 +1,11 @@
+# Node / JavaScript
+node_modules/
+dist/
+.DS_Store
+
+# Environments
+.env
+.venv
+
+# API keys
+doubao_api.txt

screencoder/README.md

@@ -0,0 +1,89 @@
+# Screencoder
+
+### About
+This is the Screencoder Project. Screencoder generates the HTML code for a website screenshot using a modular multi-agent framework.
+
+### Project Structure
+- `main.py`: The main script to generate final HTML code for a single screenshot.
+- `UIED/`: Contains the UIED (UI Element Detection) engine for analyzing screenshots and detecting components.
+  - `run_single.py`: Python script to run UI component detection on a single image.
+- `html_generator.py`: Takes the detected component data and generates a complete HTML layout with generated code for each module.
+- `image_replacer.py`: A script to replace placeholder divs in the final HTML with actual cropped images.
+- `mapping.py`: Maps the detected UIED components to logical page regions.
+- `requirements.txt`: Lists all the necessary Python dependencies for the project.
+- `doubao_api.txt`: API key file for the Doubao model (should be kept private and is included in `.gitignore`).
+
+### Setup and Installation
+
+1.  **Clone the repository:**
+    ```bash
+    git clone https://github.com/JimmyZhengyz/screencoder.git
+    cd screencoder
+    ```
+
+2.  **Create a virtual environment:**
+    ```bash
+    python3 -m venv .venv
+    source .venv/bin/activate
+    ```
+
+3.  **Install dependencies:**
+    ```bash
+    pip install -r requirements.txt
+    ```
+
+4.  **Set up API Key:**
+    - Create a file named `doubao_api.txt` in the root directory.
+    - Paste your Doubao API key into this file.
+
+### Usage
+
+The typical workflow is a multi-step process as follows:
+
+1.  **Initial Generation with Placeholders:**
+    Run the Python script to generate the initial HTML code for a given screenshot.
+    - Block Detection:
+      ```bash
+      python block_parsor.py
+      ```
+    - Generation with Placeholders (Gray Images Blocks):
+      ```bash
+      python html_generator.py
+      ```
+
+2.  **Final HTML Code:**
+    Run the python script to generate final HTML code with copped images from the original screenshot.
+    - Placeholder Detection:
+      ```bash
+      python image_box_detection.py
+      ```
+    - UI Element Detection:
+      ```bash
+      python UIED/run_single.py
+      ```
+    - Mapping Alignment Between Placeholders and UI Elements:
+      ```bash
+      python mapping.py
+      ```
+    - Placeholder Replacement:
+      ```bash
+      python image_replacer.py
+      ```
+
+3.  **Simple Run:**
+    Run the python script to generate the final HTML code:
+    ```bash
+    python main.py
+    ```
+
+### Demo
+
+To access demo, you can follow these steps:
+
+```bash
+cd demo
+pnpm install 
+pnpm run dev
+```
+
+Then, you can see the demo running.

screencoder/UIED/.idea/UIED.iml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/resnet" isTestSource="false" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="py.test" />
+  </component>
+</module>

screencoder/UIED/.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,29 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="DuplicatedCode" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <Languages>
+        <language minSize="54" name="Python" />
+      </Languages>
+    </inspection_tool>
+    <inspection_tool class="PyInterpreterInspection" enabled="false" level="WARNING" enabled_by_default="false" />
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="3">
+            <item index="0" class="java.lang.String" itemvalue="Tensorflow" />
+            <item index="1" class="java.lang.String" itemvalue="Sklearn" />
+            <item index="2" class="java.lang.String" itemvalue="Opencv" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyPep8Inspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredErrors">
+        <list>
+          <option value="E501" />
+        </list>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

screencoder/UIED/.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6" project-jdk-type="Python SDK" />
+</project>

screencoder/UIED/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/UIED.iml" filepath="$PROJECT_DIR$/.idea/UIED.iml" />
+    </modules>
+  </component>
+</project>

screencoder/UIED/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

screencoder/UIED/.idea/workspace.xml

@@ -0,0 +1,336 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="AutoImportSettings">
+    <option name="autoReloadType" value="SELECTIVE" />
+  </component>
+  <component name="ChangeListManager">
+    <list default="true" id="b4069649-920d-465f-ac6b-bac85007c2bb" name="Default" comment="">
+      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
+    </list>
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="FileTemplateManagerImpl">
+    <option name="RECENT_TEMPLATES">
+      <list>
+        <option value="Python Script" />
+      </list>
+    </option>
+  </component>
+  <component name="Git.Settings">
+    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
+  </component>
+  <component name="JupyterTrust" id="2ce0fe3c-0081-4dca-a6a6-b1219d650764" />
+  <component name="ProjectId" id="1dwCihBTog6GQX95sgxr7TpM6ZO" />
+  <component name="ProjectLevelVcsManager">
+    <ConfirmationsSetting value="2" id="Add" />
+  </component>
+  <component name="ProjectViewState">
+    <option name="hideEmptyMiddlePackages" value="true" />
+    <option name="showLibraryContents" value="true" />
+    <option name="showMembers" value="true" />
+  </component>
+  <component name="PropertiesComponent">
+    <property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
+    <property name="RunOnceActivity.ShowReadmeOnStart" value="true" />
+    <property name="WebServerToolWindowFactoryState" value="false" />
+    <property name="last_opened_file_path" value="$PROJECT_DIR$" />
+    <property name="node.js.detected.package.eslint" value="true" />
+    <property name="node.js.detected.package.tslint" value="true" />
+    <property name="node.js.path.for.package.eslint" value="project" />
+    <property name="node.js.path.for.package.tslint" value="project" />
+    <property name="node.js.selected.package.eslint" value="(autodetect)" />
+    <property name="node.js.selected.package.tslint" value="(autodetect)" />
+    <property name="restartRequiresConfirmation" value="false" />
+    <property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
+    <property name="two.files.diff.last.used.file" value="$PROJECT_DIR$/../UI2CODE/Element-Detection/merge.py" />
+  </component>
+  <component name="RecentsManager">
+    <key name="CopyFile.RECENT_KEYS">
+      <recent name="D:\git_file\github\doing\UIED\data\demo" />
+    </key>
+    <key name="MoveFile.RECENT_KEYS">
+      <recent name="D:\git_file\github\doing\UIED\detect_compo" />
+      <recent name="D:\git_file\github\doing\UIED\detect_compo\deprecated" />
+      <recent name="D:\git_file\github\doing\UIED\utils" />
+      <recent name="D:\git_file\github\doing\UIED" />
+      <recent name="D:\git_file\github\doing\UIED\result_processing" />
+    </key>
+  </component>
+  <component name="RunManager" selected="Python.run_single">
+    <configuration name="merge" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+      <module name="UIED" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/detect_merge" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/detect_merge/merge.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <configuration name="merge2" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+      <module name="UIED" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/merge2.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <configuration name="run_single" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+      <module name="UIED" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/run_single.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <configuration name="run_testing(Used for Adjusting)" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+      <module name="UIED" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/run_testing(Used for Adjusting).py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <configuration name="text_detection" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+      <module name="UIED" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/detect_text" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/detect_text/text_detection.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <list>
+      <item itemvalue="Python.merge2" />
+      <item itemvalue="Python.merge" />
+      <item itemvalue="Python.run_single" />
+      <item itemvalue="Python.run_testing(Used for Adjusting)" />
+      <item itemvalue="Python.text_detection" />
+    </list>
+    <recent_temporary>
+      <list>
+        <item itemvalue="Python.run_single" />
+        <item itemvalue="Python.run_testing(Used for Adjusting)" />
+        <item itemvalue="Python.merge" />
+        <item itemvalue="Python.merge2" />
+        <item itemvalue="Python.text_detection" />
+      </list>
+    </recent_temporary>
+  </component>
+  <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
+  <component name="SvnConfiguration">
+    <configuration />
+  </component>
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="b4069649-920d-465f-ac6b-bac85007c2bb" name="Default" comment="" />
+      <created>1581826543105</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1581826543105</updated>
+      <workItem from="1593326898629" duration="4378000" />
+      <workItem from="1594035929667" duration="39000" />
+      <workItem from="1594085137817" duration="1275000" />
+      <workItem from="1594100418101" duration="3473000" />
+      <workItem from="1594163935256" duration="1077000" />
+      <workItem from="1594182213679" duration="2275000" />
+      <workItem from="1595462115111" duration="3597000" />
+      <workItem from="1595466120556" duration="3158000" />
+      <workItem from="1595488240502" duration="74000" />
+      <workItem from="1595723287741" duration="773000" />
+      <workItem from="1596417993986" duration="2972000" />
+      <workItem from="1596442796733" duration="9601000" />
+      <workItem from="1596604406529" duration="10000" />
+      <workItem from="1596685531295" duration="1526000" />
+      <workItem from="1596792120495" duration="27485000" />
+      <workItem from="1596926803756" duration="894000" />
+      <workItem from="1596955557805" duration="717000" />
+      <workItem from="1597015667916" duration="1488000" />
+      <workItem from="1598487885922" duration="962000" />
+      <workItem from="1601609000493" duration="599000" />
+      <workItem from="1601849117964" duration="1226000" />
+      <workItem from="1601850762269" duration="6477000" />
+      <workItem from="1601933366434" duration="6847000" />
+      <workItem from="1602130037944" duration="4303000" />
+      <workItem from="1602199380249" duration="5493000" />
+      <workItem from="1603669721746" duration="6042000" />
+      <workItem from="1604011435077" duration="517000" />
+      <workItem from="1604016832655" duration="5114000" />
+      <workItem from="1604037397074" duration="12109000" />
+      <workItem from="1604564719252" duration="1133000" />
+      <workItem from="1604619358289" duration="17569000" />
+      <workItem from="1604874207809" duration="171000" />
+      <workItem from="1605071062104" duration="3079000" />
+      <workItem from="1605086142565" duration="6397000" />
+      <workItem from="1625010508533" duration="4541000" />
+      <workItem from="1625099073176" duration="49720000" />
+      <workItem from="1625443415902" duration="26350000" />
+      <workItem from="1625529598285" duration="26479000" />
+      <workItem from="1625613709029" duration="14000" />
+      <workItem from="1625730508694" duration="928000" />
+      <workItem from="1625809233064" duration="837000" />
+      <workItem from="1626009011038" duration="18000" />
+      <workItem from="1626307428798" duration="1983000" />
+      <workItem from="1628054466383" duration="1894000" />
+      <workItem from="1628122812217" duration="7049000" />
+      <workItem from="1630237947629" duration="453000" />
+      <workItem from="1630268189943" duration="20000" />
+      <workItem from="1630297231550" duration="5710000" />
+      <workItem from="1630312264694" duration="7894000" />
+      <workItem from="1631149307515" duration="752000" />
+      <workItem from="1631576239206" duration="805000" />
+      <workItem from="1631584649434" duration="2272000" />
+      <workItem from="1648024033251" duration="1883000" />
+    </task>
+    <servers />
+  </component>
+  <component name="TestHistory">
+    <history-entry file="py_test_in_test_lucky_py - 2020.03.04 at 05h 51m 04s.xml">
+      <configuration name="py.test in test_lucky.py" configurationId="tests" />
+    </history-entry>
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="3" />
+  </component>
+  <component name="XDebuggerManager">
+    <breakpoint-manager>
+      <breakpoints>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url />
+          <line>147</line>
+          <option name="timeStamp" value="3" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url />
+          <line>134</line>
+          <option name="timeStamp" value="4" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url />
+          <line>135</line>
+          <option name="timeStamp" value="5" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url />
+          <line>136</line>
+          <option name="timeStamp" value="6" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/detect_text_east/lib_east/eval.py</url>
+          <line>263</line>
+          <option name="timeStamp" value="67" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/result_processing/eval_classes.py</url>
+          <line>92</line>
+          <option name="timeStamp" value="92" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/detect_text_east/lib_east/eval.py</url>
+          <line>108</line>
+          <option name="timeStamp" value="93" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/cnn/CNN.py</url>
+          <line>62</line>
+          <option name="timeStamp" value="94" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/run_single.py</url>
+          <line>27</line>
+          <option name="timeStamp" value="101" />
+        </line-breakpoint>
+      </breakpoints>
+      <default-breakpoints>
+        <breakpoint type="python-exception">
+          <properties notifyOnTerminate="true" exception="BaseException">
+            <option name="notifyOnTerminate" value="true" />
+          </properties>
+        </breakpoint>
+      </default-breakpoints>
+    </breakpoint-manager>
+  </component>
+  <component name="com.intellij.coverage.CoverageDataManagerImpl">
+    <SUITE FILE_PATH="coverage/UIED$view_gt.coverage" NAME="view_gt Coverage Results" MODIFIED="1596418105849" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/result_processing" />
+    <SUITE FILE_PATH="coverage/UIED$run_single.coverage" NAME="run_single Coverage Results" MODIFIED="1648024874423" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/UIED$main_single.coverage" NAME="main_single Coverage Results" MODIFIED="1594102734340" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/UIED$run_batch.coverage" NAME="run_batch Coverage Results" MODIFIED="1596448499254" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/UIED_block$main_single.coverage" NAME="main_single Coverage Results" MODIFIED="1594035942350" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/UIED$run_testing_Used_for_Adjusting_.coverage" NAME="run_testing(Used for Adjusting) Coverage Results" MODIFIED="1631149318891" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/UIED$merge2.coverage" NAME="merge2 Coverage Results" MODIFIED="1625271385465" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/UIED$run_testing_Use_Me_for_Adjusting_.coverage" NAME="run_testing(Use Me for Adjusting) Coverage Results" MODIFIED="1605091364353" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/UIED$text_detection.coverage" NAME="text_detection Coverage Results" MODIFIED="1625107691453" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/detect_text" />
+    <SUITE FILE_PATH="coverage/UIED$merge.coverage" NAME="merge Coverage Results" MODIFIED="1625284362497" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/detect_merge" />
+    <SUITE FILE_PATH="coverage/UIED$experiment.coverage" NAME="experiment Coverage Results" MODIFIED="1605087951044" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/result_processing" />
+  </component>
+</project>

screencoder/UIED/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [2021] [UIED [email protected]]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

screencoder/UIED/README.md

@@ -0,0 +1,80 @@
+# UIED - UI element detection, detecting UI elements from UI screenshots or drawnings
+
+This project is still ongoing and this repo may be updated irregularly, I developed a web app for the UIED in http://uied.online
+
+## Related Publications: 
+[1. UIED: a hybrid tool for GUI element detection](https://dl.acm.org/doi/10.1145/3368089.3417940)
+
+[2. Object Detection for Graphical User Interface: Old Fashioned or Deep Learning or a Combination?](https://arxiv.org/abs/2008.05132)
+
+>The repo has been **upgraded with Google OCR** for GUI text detection, to use the original version in our paper (using [EAST](https://github.com/argman/EAST) as text detector), check the relase [v2.3](https://github.com/MulongXie/UIED/releases/tag/v2.3) and download the pre-trained model in [this link](https://drive.google.com/drive/folders/1MK0Om7Lx0wRXGDfNcyj21B0FL1T461v5?usp=sharing).
+
+## What is it?
+
+UI Element Detection (UIED) is an old-fashioned computer vision (CV) based element detection approach for graphic user interface. 
+
+The input of UIED could be various UI image, such as mobile app or web page screenshot, UI design drawn by Photoshop or Sketch, and even some hand-drawn UI design. Then the approach detects and classifies text and graphic UI elements, and exports the detection result as JSON file for future application. 
+
+UIED comprises two parts to detect UI text and graphic elements, such as button, image and input bar. 
+* For text, it leverages [Google OCR](https://cloud.google.com/vision/docs/ocr) to perfrom detection. 
+
+* For graphical elements, it uses old-fashioned CV approaches to locate the elements and a CNN classifier to achieve classification. 
+
+> UIED is highly customizable, you can replace both parts by your choice (e.g. other text detection approaches). Unlike black-box end-to-end deep learning approach, you can revise the algorithms in the non-text detection and merging (partially or entirely) easily to fit your task.
+
+![UIED Approach](https://github.com/MulongXie/UIED/blob/master/data/demo/approach.png)
+
+## How to use?
+
+### Dependency
+* **Python 3.5**
+* **Opencv 3.4.2**
+* **Pandas**
+<!-- * **Tensorflow 1.10.0**
+* **Keras 2.2.4**
+* **Sklearn 0.22.2** -->
+
+### Installation
+<!-- Install the mentioned dependencies, and download two pre-trained models from [this link](https://drive.google.com/drive/folders/1MK0Om7Lx0wRXGDfNcyj21B0FL1T461v5?usp=sharing) for EAST text detection and GUI element classification. -->
+
+<!-- Change ``CNN_PATH`` and ``EAST_PATH`` in *config/CONFIG.py* to your locations. -->
+
+The new version of UIED equipped with Google OCR is easy to deploy and no pre-trained model is needed. Simply donwload the repo along with the dependencies.
+
+> Please replace the Google OCR key at `detect_text/ocr.py line 28` with your own (apply in [Google website](https://cloud.google.com/vision)).
+
+### Usage
+To test your own image(s):
+* To test single image, change *input_path_img* in ``run_single.py`` to your input image and the results will be output to *output_root*.
+* To test mutiple images, change *input_img_root* in ``run_batch.py`` to your input directory and the results will be output to *output_root*.
+* To adjust the parameters lively, using ``run_testing.py`` 
+
+> Note: The best set of parameters vary for different types of GUI image (Mobile App, Web, PC). I highly recommend to first play with the ``run_testing.py`` to pick a good set of parameters for your data.
+   
+## Folder structure
+``cnn/``
+* Used to train classifier for graphic UI elements
+* Set path of the CNN classification model
+
+``config/``
+* Set data paths 
+* Set parameters for graphic elements detection
+
+``data/``
+* Input UI images and output detection results
+
+``detect_compo/``
+* Non-text GUI component detection
+
+``detect_text/``
+* GUI text detection using Google OCR
+
+``detect_merge/``
+* Merge the detection results of non-text and text GUI elements
+
+The major detection algorithms are in ``detect_compo/``, ``detect_text/`` and ``detect_merge/``
+
+## Demo
+GUI element detection result for web screenshot
+ 
+![UI Components detection result](https://github.com/MulongXie/UIED/blob/master/data/demo/demo.png)

screencoder/UIED/cnn/CNN.py

@@ -0,0 +1,125 @@
+import keras
+from keras.applications.resnet50 import ResNet50
+from keras.models import Model,load_model
+from keras.layers import Dense, Activation, Flatten, Dropout
+from sklearn.metrics import confusion_matrix
+import numpy as np
+import cv2
+
+from config.CONFIG import Config
+cfg = Config()
+
+
+class CNN:
+    def __init__(self, classifier_type, is_load=True):
+        '''
+        :param classifier_type: 'Text' or 'Noise' or 'Elements'
+        '''
+        self.data = None
+        self.model = None
+
+        self.classifier_type = classifier_type
+
+        self.image_shape = (32,32,3)
+        self.class_number = None
+        self.class_map = None
+        self.model_path = None
+        self.classifier_type = classifier_type
+        if is_load:
+            self.load(classifier_type)
+
+    def build_model(self, epoch_num, is_compile=True):
+        base_model = ResNet50(include_top=False, weights='imagenet', input_shape=self.image_shape)
+        for layer in base_model.layers:
+            layer.trainable = False
+        self.model = Flatten()(base_model.output)
+        self.model = Dense(128, activation='relu')(self.model)
+        self.model = Dropout(0.5)(self.model)
+        self.model = Dense(15, activation='softmax')(self.model)
+
+        self.model = Model(inputs=base_model.input, outputs=self.model)
+        if is_compile:
+            self.model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy'])
+            self.model.fit(self.data.X_train, self.data.Y_train, batch_size=64, epochs=epoch_num, verbose=1,
+                           validation_data=(self.data.X_test, self.data.Y_test))
+
+    def train(self, data, epoch_num=30):
+        self.data = data
+        self.build_model(epoch_num)
+        self.model.save(self.model_path)
+        print("Trained model is saved to", self.model_path)
+
+    def load(self, classifier_type):
+        if classifier_type == 'Text':
+            self.model_path = 'E:/Mulong/Model/rico_compos/cnn-textview-2.h5'
+            self.class_map = ['Text', 'Non-Text']
+        elif classifier_type == 'Noise':
+            self.model_path = 'E:/Mulong/Model/rico_compos/cnn-noise-1.h5'
+            self.class_map = ['Noise', 'Non-Noise']
+        elif classifier_type == 'Elements':
+            # self.model_path = 'E:/Mulong/Model/rico_compos/resnet-ele14-19.h5'
+            # self.model_path = 'E:/Mulong/Model/rico_compos/resnet-ele14-28.h5'
+            # self.model_path = 'E:/Mulong/Model/rico_compos/resnet-ele14-45.h5'
+            self.model_path = 'UIED/cnn/model/cnn-rico-1.h5' # Use local model
+            self.class_map = cfg.element_class
+            self.image_shape = (64, 64, 3)
+        elif classifier_type == 'Image':
+            # Redirect 'Image' classification to use the general 'Elements' model
+            # as the specific model is not available in the project.
+            # IMPORTANT: This requires the actual model file to be present for real classification.
+            print("Warning: 'Image' specific model not found. Redirecting to general 'Elements' classifier.")
+            self.model_path = 'UIED/cnn/model/cnn-rico-1.h5' # Use local model
+            self.class_map = ['Image', 'Non-Image'] # Keep the class map for binary classification logic
+
+        self.class_number = len(self.class_map)
+        try:
+            self.model = load_model(self.model_path)
+            print('Model Loaded From', self.model_path)
+        except Exception as e:
+            print(f"Error loading model: {e}")
+            print("A dummy model file was created, but it's not a valid Keras model.")
+            print("Please replace it with the actual model file for classification to work.")
+            self.model = None
+
+    def preprocess_img(self, image):
+        image = cv2.resize(image, self.image_shape[:2])
+        x = (image / 255).astype('float32')
+        x = np.array([x])
+        return x
+
+    def predict(self, imgs, compos, load=False, show=False):
+        """
+        :type img_path: list of img path
+        """
+        if load:
+            self.load(self.classifier_type)
+        if self.model is None:
+            print("*** No model loaded ***")
+            return
+        for i in range(len(imgs)):
+            X = self.preprocess_img(imgs[i])
+            Y = self.class_map[np.argmax(self.model.predict(X))]
+            compos[i].category = Y
+            if show:
+                print(Y)
+                cv2.imshow('element', imgs[i])
+                cv2.waitKey()
+
+    def evaluate(self, data, load=True):
+        if load:
+            self.load(self.classifier_type)
+        X_test = data.X_test
+        Y_test = [np.argmax(y) for y in data.Y_test]
+        Y_pre = [np.argmax(y_pre) for y_pre in self.model.predict(X_test, verbose=1)]
+
+        matrix = confusion_matrix(Y_test, Y_pre)
+        print(matrix)
+
+        TP, FP, FN = 0, 0, 0
+        for i in range(len(matrix)):
+            TP += matrix[i][i]
+            FP += sum(matrix[i][:]) - matrix[i][i]
+            FN += sum(matrix[:][i]) - matrix[i][i]
+        precision = TP/(TP+FP)
+        recall = TP / (TP+FN)
+        print("Precision:%.3f, Recall:%.3f" % (precision, recall))

screencoder/UIED/cnn/Config.py

@@ -0,0 +1,21 @@
+
+class Config:
+    def __init__(self):
+        # cnn 4 classes
+        # self.MODEL_PATH = 'E:/Mulong/Model/ui_compos/cnn6_icon.h5'   # cnn 4 classes
+        # self.class_map = ['Image', 'Icon', 'Button', 'Input']
+
+        # resnet 14 classes
+        # self.DATA_PATH = "E:/Mulong/Datasets/rico/elements-14-2"
+        # self.MODEL_PATH = 'E:/Mulong/Model/rico_compos/resnet-ele14.h5'
+        # self.class_map = ['Button', 'CheckBox', 'Chronometer', 'EditText', 'ImageButton', 'ImageView',
+        #                   'ProgressBar', 'RadioButton', 'RatingBar', 'SeekBar', 'Spinner', 'Switch',
+        #                   'ToggleButton', 'VideoView', 'TextView']  # ele-14
+
+        self.DATA_PATH = "E:\Mulong\Datasets\dataset_webpage\Components3"
+
+        self.MODEL_PATH = 'E:/Mulong/Model/rico_compos/cnn2-textview.h5'
+        self.class_map = ['Text', 'Non-Text']
+
+        self.image_shape = (32, 32, 3)
+        self.class_number = len(self.class_map)

screencoder/UIED/cnn/Data.py

@@ -0,0 +1,69 @@
+import cv2
+import numpy as np
+from os.path import join as pjoin
+import glob
+from tqdm import tqdm
+from Config import Config
+
+cfg = Config()
+
+
+class Data:
+    def __init__(self):
+        self.data_num = 0
+        self.images = []
+        self.labels = []
+        self.X_train, self.Y_train = None, None
+        self.X_test, self.Y_test = None, None
+
+        self.image_shape = cfg.image_shape
+        self.class_number = cfg.class_number
+        self.class_map = cfg.class_map
+        self.DATA_PATH = cfg.DATA_PATH
+
+    def load_data(self, resize=True, shape=None, max_number=1000000):
+        # if customize shape
+        if shape is not None:
+            self.image_shape = shape
+        else:
+            shape = self.image_shape
+
+        # load data
+        for p in glob.glob(pjoin(self.DATA_PATH, '*')):
+            print("*** Loading components of %s: %d ***" %(p.split('\\')[-1], int(len(glob.glob(pjoin(p, '*.png'))))))
+            label = self.class_map.index(p.split('\\')[-1])  # map to index of classes
+            for i, image_path in enumerate(tqdm(glob.glob(pjoin(p, '*.png'))[:max_number])):
+                image = cv2.imread(image_path)
+                if resize:
+                    image = cv2.resize(image, shape[:2])
+                self.images.append(image)
+                self.labels.append(label)
+
+        assert len(self.images) == len(self.labels)
+        self.data_num = len(self.images)
+        print('%d Data Loaded' % self.data_num)
+
+    def generate_training_data(self, train_data_ratio=0.8):
+        # transfer int into c dimensions one-hot array
+        def expand(label, class_number):
+            # return y : (num_class, num_samples)
+            y = np.eye(class_number)[label]
+            y = np.squeeze(y)
+            return y
+
+        # reshuffle
+        np.random.seed(0)
+        self.images = np.random.permutation(self.images)
+        np.random.seed(0)
+        self.labels = np.random.permutation(self.labels)
+        Y = expand(self.labels, self.class_number)
+
+        # separate dataset
+        cut = int(train_data_ratio * self.data_num)
+        self.X_train = (self.images[:cut] / 255).astype('float32')
+        self.X_test = (self.images[cut:] / 255).astype('float32')
+        self.Y_train = Y[:cut]
+        self.Y_test = Y[cut:]
+
+        print('X_train:%d, Y_train:%d' % (len(self.X_train), len(self.Y_train)))
+        print('X_test:%d, Y_test:%d' % (len(self.X_test), len(self.Y_test)))

screencoder/UIED/config/CONFIG.py

@@ -0,0 +1,45 @@
+from os.path import join as pjoin
+import os
+
+
+class Config:
+
+    def __init__(self):
+        # setting CNN (graphic elements) model
+        self.image_shape = (64, 64, 3)
+        # self.MODEL_PATH = 'E:\\Mulong\\Model\\UI2CODE\\cnn6_icon.h5'
+        # self.class_map = ['button', 'input', 'icon', 'img', 'text']
+        self.CNN_PATH = 'E:/Mulong/Model/rico_compos/cnn-rico-1.h5'
+        self.element_class = ['Button', 'CheckBox', 'Chronometer', 'EditText', 'ImageButton', 'ImageView',
+                              'ProgressBar', 'RadioButton', 'RatingBar', 'SeekBar', 'Spinner', 'Switch',
+                              'ToggleButton', 'VideoView', 'TextView']
+        self.class_number = len(self.element_class)
+
+        # setting EAST (ocr) model
+        self.EAST_PATH = 'E:/Mulong/Model/East/east_icdar2015_resnet_v1_50_rbox'
+
+        self.COLOR = {'Button': (0, 255, 0), 'CheckBox': (0, 0, 255), 'Chronometer': (255, 166, 166),
+                      'EditText': (255, 166, 0),
+                      'ImageButton': (77, 77, 255), 'ImageView': (255, 0, 166), 'ProgressBar': (166, 0, 255),
+                      'RadioButton': (166, 166, 166),
+                      'RatingBar': (0, 166, 255), 'SeekBar': (0, 166, 10), 'Spinner': (50, 21, 255),
+                      'Switch': (80, 166, 66), 'ToggleButton': (0, 66, 80), 'VideoView': (88, 66, 0),
+                      'TextView': (169, 255, 0), 'NonText': (0,0,255),
+                      'Compo':(0, 0, 255), 'Text':(169, 255, 0), 'Block':(80, 166, 66)}
+
+    def build_output_folders(self):
+        # setting data flow paths
+        self.ROOT_INPUT = "E:\\Mulong\\Datasets\\rico\\combined"
+        self.ROOT_OUTPUT = "E:\\Mulong\\Result\\rico\\rico_uied\\rico_new_uied_v3"
+
+        self.ROOT_IMG_ORG = pjoin(self.ROOT_INPUT, "org")
+        self.ROOT_IP = pjoin(self.ROOT_OUTPUT, "ip")
+        self.ROOT_OCR = pjoin(self.ROOT_OUTPUT, "ocr")
+        self.ROOT_MERGE = pjoin(self.ROOT_OUTPUT, "merge")
+        self.ROOT_IMG_COMPONENT = pjoin(self.ROOT_OUTPUT, "components")
+        if not os.path.exists(self.ROOT_IP):
+            os.mkdir(self.ROOT_IP)
+        if not os.path.exists(self.ROOT_OCR):
+            os.mkdir(self.ROOT_OCR)
+        if not os.path.exists(self.ROOT_MERGE):
+            os.mkdir(self.ROOT_MERGE)

screencoder/UIED/config/CONFIG_UIED.py

@@ -0,0 +1,49 @@
+class Config:
+
+    def __init__(self):
+        # Adjustable
+        # self.THRESHOLD_PRE_GRADIENT = 4             # dribbble:4 rico:4 web:1
+        # self.THRESHOLD_OBJ_MIN_AREA = 55            # bottom line 55 of small circle
+        # self.THRESHOLD_BLOCK_GRADIENT = 5
+
+        # *** Frozen ***
+        self.THRESHOLD_REC_MIN_EVENNESS = 0.7
+        self.THRESHOLD_REC_MAX_DENT_RATIO = 0.25
+        self.THRESHOLD_LINE_THICKNESS = 8
+        self.THRESHOLD_LINE_MIN_LENGTH = 0.95
+        self.THRESHOLD_COMPO_MAX_SCALE = (0.25, 0.98)  # (120/800, 422.5/450) maximum height and width ratio for a atomic compo (button)
+        self.THRESHOLD_TEXT_MAX_WORD_GAP = 10
+        self.THRESHOLD_TEXT_MAX_HEIGHT = 0.04  # 40/800 maximum height of text
+        self.THRESHOLD_TOP_BOTTOM_BAR = (0.045, 0.94)  # (36/800, 752/800) height ratio of top and bottom bar
+        self.THRESHOLD_BLOCK_MIN_HEIGHT = 0.03  # 24/800
+
+        # deprecated
+        # self.THRESHOLD_OBJ_MIN_PERIMETER = 0
+        # self.THRESHOLD_BLOCK_MAX_BORDER_THICKNESS = 8
+        # self.THRESHOLD_BLOCK_MAX_CROSS_POINT = 0.1
+        # self.THRESHOLD_UICOMPO_MIN_W_H_RATIO = 0.4
+        # self.THRESHOLD_TEXT_MAX_WIDTH = 150
+        # self.THRESHOLD_LINE_MIN_LENGTH_H = 50
+        # self.THRESHOLD_LINE_MIN_LENGTH_V = 50
+        # self.OCR_PADDING = 5
+        # self.OCR_MIN_WORD_AREA = 0.45
+        # self.THRESHOLD_MIN_IOU = 0.1              # dribbble:0.003 rico:0.1 web:0.1
+        # self.THRESHOLD_BLOCK_MIN_EDGE_LENGTH = 210   # dribbble:68 rico:210 web:70
+        # self.THRESHOLD_UICOMPO_MAX_W_H_RATIO = 10   # dribbble:10 rico:10 web:22
+
+        self.CLASS_MAP = {'0':'Button', '1':'CheckBox', '2':'Chronometer', '3':'EditText', '4':'ImageButton', '5':'ImageView',
+               '6':'ProgressBar', '7':'RadioButton', '8':'RatingBar', '9':'SeekBar', '10':'Spinner', '11':'Switch',
+               '12':'ToggleButton', '13':'VideoView', '14':'TextView'}
+        self.COLOR = {'Button': (0, 255, 0), 'CheckBox': (0, 0, 255), 'Chronometer': (255, 166, 166),
+                      'EditText': (255, 166, 0),
+                      'ImageButton': (77, 77, 255), 'ImageView': (255, 0, 166), 'ProgressBar': (166, 0, 255),
+                      'RadioButton': (166, 166, 166),
+                      'RatingBar': (0, 166, 255), 'SeekBar': (0, 166, 10), 'Spinner': (50, 21, 255),
+                      'Switch': (80, 166, 66), 'ToggleButton': (0, 66, 80), 'VideoView': (88, 66, 0),
+                      'TextView': (169, 255, 0),
+
+                      'Text':(169, 255, 0), 'Non-Text':(255, 0, 166),
+
+                      'Noise':(6,6,255), 'Non-Noise': (6,255,6),
+
+                      'Image':(255,6,6), 'Non-Image':(6,6,255)}