init commit

.gitattributes

@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+robotiq_2f_85/ filter=lfs diff=lfs merge=lfs -text
+*.stl filter=lfs diff=lfs merge=lfs -text
+*.urdf filter=lfs diff=lfs merge=lfs -text
+*.mtl filter=lfs diff=lfs merge=lfs -text
+*.obj filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.dae filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,164 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+.DS_Store
+
+.vscode/

LICENSE.md

@@ -0,0 +1,7 @@
+Copyright 2021 Google LLC. SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+
+https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

README.md

@@ -1,12 +1,69 @@
----
-title: VoxPoserExamples
-emoji: 🔥
-colorFrom: red
-colorTo: green
-sdk: gradio
-sdk_version: 3.40.1
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# VoxPoser API Examples
+
+## Usage
+
+```bash
+python3 app.py
+```
+
+1. 在界面中填写OpenAI API Key，使用的代理地址，选择需要的configuration
+2. 点击Setup/Reset Simulation
+3. 填写自定义Instruction
+4. 点击Run执行（需要等待较长时间）
+
+## Example
+
+### VLM & Perception
+
+1. Open Vocab object detection [owlvit](https://huggingface.co/docs/transformers/model_doc/owlvit)
+2. [SAM](https://github.com/facebookresearch/segment-anything)
+3. Object mask tracking [XMem](https://github.com/hkchengrex/XMem)
+4. 使用realsense获得深度图
+5. 使用深度图获得法向量（抓取位姿）
+
+可替代性：
+- [x] owlvit -> Grounded SAM / YOLO
+- [x] SAM -> FastSAM / YOLO-seg
+- [ ] XMem -> DeepSORT(?) ByteTrack(?)
+
+### LMP语言模型编程
+
+语言模型编程：使用GPT-4
+
+VoxPoser需要三大类LMP:
+1. Planner
+2. Composer
+3. Value map generator
+
+可替代性：
+- [ ] GPT-4 -> LLaMA2 (?)
+
+## LMPs
+
+### Planner
+
+LMP的输出是一系列的编程模型接口，Planner将这些语言描述转化为一系列高层级的规划，每步规划这些动作将被Composer执行。
+
+模拟环境中不使用规划器，因为评估的任务由单个操作阶段组成。
+
+### Composer
+
+Composer LMP 从依次逐渐调用如下模组：
+1. 感知模组调用获得感知结果
+2. [optional] Affordance LMP
+3. [optional] Avoidance LMP
+4. [optional] End Effector Velocity LMP
+5. [optional] End Effector Rotation LMP
+6. [optional] Gripper Action LMP
+7. Execute
+
+### Value Maps
+
+TODO
+
+### Execution
+
+1. Motion Planner: 贪心搜索得到一系列末端位姿，仅适用Affordance Map 和 Avoidance Map
+2. Cost map: $W = -2 * \text{norm}(\text{Affordance}) - \text{norm}(\text{Avoidance})$
+3. 根据离开/接近，调用目标法向量的正/负值方向上的Affordance Map
+4. 根据避障目标的占据栅格occupancy_map，调整Avoidance Map

README_CaP.md

@@ -0,0 +1,53 @@
+---
+title: Code As Policies
+emoji: 📈
+colorFrom: purple
+colorTo: indigo
+sdk: gradio
+sdk_version: 3.3.1
+app_file: app.py
+pinned: false
+license: apache-2.0
+---
+
+# Code as Policies Tabletop Manipulation Interactive Demo
+
+This demo is from the paper:
+
+[Code as Policies: Language Model Programs for Embodied Control](https://code-as-policies.github.io/)
+
+Below is an interactive demo for the simulated tabletop manipulation domain, seen in the paper section IV.D
+
+## Preparations
+1. Obtain an [OpenAI API Key](https://openai.com/blog/openai-api/)
+
+## Usage 
+1. Fill in the API Key and how many blocks and bowls to be spawned in the environment. 
+2. Click Setup/Reset Simulation
+3. Based on the new randomly sampled object names, input an instruction and click Run Instruction. If successful, this will render a video and update the simulation environment visualization.
+
+You can run instructions in sequence and refer back to previous instructions (e.g. do the same with other blocks, move the same block to the other bowl, etc). To reset, click Setup/Reset Env, and this will clear the current instruction history.
+
+## Supported Instructions
+* Spatial reasoning (e.g. to the left of the red block, the closest corner, the farthest bowl, the second block from the right)
+* Sequential actions (e.g. put blocks in matching bowls, stack blocks on the bottom right corner)
+* Contextual instructions (e.g. do the same with the blue block, undo that)
+* Language-based reasoning (e.g. put the forest-colored block on the ocean-colored bowl).
+* Simple Q&A (e.g. how many blocks are to the left of the blue bowl?)
+
+## Example Instructions 
+Note object names may need to be changed depending the sampled object names.
+* put the sun-colored block on the bowl closest to it
+* stack the blocks on the bottom most bowl
+* arrange the blocks as a square in the middle
+* move the square 5cm to the right
+* how many blocks are to the right of the orange bowl?
+* pick up the block closest to the top left corner and place it on the bottom right corner
+
+## Known Limitations
+* In simulation we're using ground truth object poses instead of using vision models. This means that instructions the require knowledge of visual apperances (e.g. darkest bowl, largest object, empty bowls) are not supported.
+* Currently, the low-level pick place primitive does not do collision checking, so if there are many objects on the table, placing actions may incur collisions.
+* The pick place primitive is also unable to pick up bowls.
+* Prompt saturation - if too many instructions (10+) are executed in a row, then the LLM may start to ignore examples in the early parts of the prompt.
+* Ambiguous instructions - if a given instruction doesn't lead to the desired actions, try rephrasing it to remove ambiguities (e.g. place the block on the closest bowl -> place the block on its closest bowl)
+* Maximum token length - you may hit the maximum token length if running multiple commands in sequence. Please reset the simulation when this happens.

Real/composer/real_composer_prompt.py

@@ -0,0 +1,120 @@
+import numpy as np
+from env_utils import execute, reset_to_default_pose
+from perception_utils import parse_query_obj
+from plan_utils import get_affordance_map, get_avoidance_map, get_velocity_map, get_rotation_map, get_gripper_map
+
+# Query: move ee forward for 10cm.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map(f'a point 10cm in front of {movable.position}')
+execute(movable, affordance_map)
+
+# Query: go back to default.
+reset_to_default_pose()
+
+# Query: move the gripper behind the bowl, and slow down when near the bowl.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point 15cm behind the bowl')
+avoidance_map = get_avoidance_map('10cm near the bowl')
+velocity_map = get_velocity_map('slow down when near the bowl')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map, velocity_map=velocity_map)
+
+# Query: move to the back side of the table while staying at least 5cm from the blue block.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point on the back side of the table')
+avoidance_map = get_avoidance_map('5cm from the blue block')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map)
+
+# Query: move to the top of the plate and face the plate.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point 10cm above the plate')
+rotation_map = get_rotation_map('face the plate')
+execute(movable, affordance_map=affordance_map, rotation_map=rotation_map)
+
+# Query: drop the toy inside container.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point 15cm above the container')
+gripper_map = get_gripper_map('close everywhere but open when on top of the container')
+execute(movable, affordance_map=affordance_map, gripper_map=gripper_map)
+
+# Query: push close the topmost drawer.
+movable = parse_query_obj('topmost drawer handle')
+affordance_map = get_affordance_map('a point 30cm into the topmost drawer handle')
+execute(movable, affordance_map=affordance_map)
+
+# Query: push the second to the left block along the red line.
+movable = parse_query_obj('second to the left block')
+affordance_map = get_affordance_map('the red line')
+execute(movable, affordance_map=affordance_map)
+
+# Query: grasp the blue block from the table at a quarter of the speed.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point at the center of blue block')
+velocity_map = get_velocity_map('quarter of the speed')
+gripper_map = get_gripper_map('open everywhere except 1cm around the blue block')
+execute(movable, affordance_map=affordance_map, velocity_map=velocity_map, gripper_map=gripper_map)
+
+# Query: move to the left of the brown block.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point 10cm to the left of the brown block')
+execute(movable, affordance_map=affordance_map)
+
+# Query: move to the top of the tray that contains the lemon.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point 10cm above the tray that contains the lemon')
+execute(movable, affordance_map=affordance_map)
+
+# Query: close drawer by 5cm.
+movable = parse_query_obj('drawer handle')
+affordance_map = get_affordance_map('a point 5cm into the drawer handle')
+execute(movable, affordance_map=affordance_map)
+
+# Query: move to 5cm on top of the soda can, at 0.5x speed when within 20cm of the wooden mug, and keep at least 15cm away from the wooden mug.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point 5cm above the soda can')
+avoidance_map = get_avoidance_map('15cm from the wooden mug')
+velocity_map = get_velocity_map('0.5x speed when within 20cm of the wooden mug')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map, velocity_map=velocity_map)
+
+# Query: wipe the red dot but avoid the blue block.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('the red dot')
+avoidance_map = get_avoidance_map('10cm from the blue block')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map)
+
+# Query: grasp the mug from the shelf.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point at the center of the mug handle')
+gripper_map = get_gripper_map('open everywhere except 1cm around the mug handle')
+execute(movable, affordance_map=affordance_map, gripper_map=gripper_map)
+
+# Query: move to 10cm on top of the soup bowl, and 5cm to the left of the soup bowl, while away from the glass, at 0.75x speed.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point 10cm above and 5cm to the left of the soup bowl')
+avoidance_map = get_avoidance_map('10cm from the glass')
+velocity_map = get_velocity_map('0.75x speed')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map, velocity_map=velocity_map)
+
+# Query: open gripper.
+movable = parse_query_obj('gripper')
+gripper_map = get_gripper_map('open everywhere')
+execute(movable, gripper_map=gripper_map)
+
+# Query: turn counter-clockwise by 180 degrees.
+movable = parse_query_obj('gripper')
+rotation_map = get_rotation_map('turn counter-clockwise by 180 degrees')
+execute(movable, rotation_map=rotation_map)
+
+# Query: sweep all particles to the left side of the table.
+particles = parse_query_obj('particles')
+for particle in particles:
+    movable = particle
+    affordance_map = get_affordance_map('a point on the left side of the table')
+    execute(particle, affordance_map=affordance_map)
+
+# Query: grasp the bottom drawer handle while moving at 0.5x speed.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point at the center of the bottom drawer handle')
+velocity_map = get_velocity_map('0.5x speed')
+rotation_map = get_rotation_map('face the bottom drawer handle')
+gripper_map = get_gripper_map('open everywhere except 1cm around the bottom drawer handle')
+execute(movable, affordance_map=affordance_map, velocity_map=velocity_map, rotation_map=rotation_map, gripper_map=gripper_map)

Real/perception/real_parse_query_obj_prompt.py

@@ -0,0 +1,104 @@
+import numpy as np
+from perception_utils import detect
+
+objects = ['table', 'gripper', 'green block', 'cardboard box']
+# Query: gripper.
+gripper = detect('gripper')[0]
+ret_val = gripper
+
+objects = ['table', 'gripper', 'drawer', 'egg', 'egg', 'plate']
+# Query: topmost handle.
+handles = detect('drawer handle')
+handles = sorted(handles, key=lambda x: x.position[2])
+top_handle = handles[-1]
+ret_val = top_handle
+
+objects = ['table', 'gripper', 'yellow block', 'charging cable', 'cyan block', 'magenta block']
+# Query: second to the left block.
+blocks = detect('block')
+blocks = sorted(blocks, key=lambda x: x.position[1])
+second_left_block = blocks[1]
+ret_val = second_left_block
+
+objects = ['table', 'gripper', 'iPhone', 'ruler', 'pink line', 'blue line']
+# Query: the front most line on the table.
+lines = detect('line')
+lines = sorted(lines, key=lambda x: x.position[0])
+front_most_line = lines[-1]
+ret_val = front_most_line
+
+objects = ['table', 'gripper', 'vase', 'napkin box', 'mask']
+# Query: table.
+table = detect('table')[0]
+ret_val = table
+
+objects = ['table', 'gripper', 'bottle', 'drawer', 'bowl', 'bag']
+# Query: second to the bottom handle.
+handles = detect('drawer handle')
+handles = sorted(handles, key=lambda x: x.position[2])
+second_bottom_handle = handles[1]
+ret_val = second_bottom_handle
+
+objects = ['table', 'gripper', 'brown line', 'red block', 'monitor']
+# Query: brown line.
+brown_line = detect('brown line')[0]
+ret_val = brown_line
+
+objects = ['table', 'gripper', 'green block', 'cup holder', 'black block']
+# Query: block.
+block = detect('green block')[0]
+ret_val = block
+
+objects = ['table', 'gripper', 'mouse', 'yellow bowl', 'brown bowl', 'sticker']
+# Query: bowl closest to the sticker.
+bowls = detect('bowl')
+sticker = detect('sticker')[0]
+closest_bowl = min(bowls, key=lambda x: np.linalg.norm(x.position - sticker.position))
+ret_val = closest_bowl
+
+objects = ['table', 'gripper', 'keyboard', 'brown bag', 'pink bag', 'red tape', 'bottle']
+# Query: bag with the red tape on top.
+bags = detect('bag')
+red_tape = detect('red tape')[0]
+bag_with_red_tape = min(bags, key=lambda x: np.linalg.norm(x.position - red_tape.position))
+ret_val = bag_with_red_tape
+
+objects = ['table', 'gripper', 'grape', 'wood tray', 'strawberry', 'white tray', 'blue tray', 'bread']
+# Query: tray that contains the bread.
+trays = detect('tray')
+bread = detect('bread')[0]
+tray_with_bread = min(trays, key=lambda x: np.linalg.norm(x.position - bread.position))
+ret_val = tray_with_bread
+
+objects = ['table', 'gripper', 'drawer']
+# Query: top drawer handle.
+handles = detect('drawer handle')
+top_drawer_handle = max(handles, key=lambda x: x.position[2])
+ret_val = top_drawer_handle
+
+objects = ['table', 'gripper', 'door']
+# Query: the thing you can open the door with.
+door_handle = detect('door handle')[0]
+ret_val = door_handle
+
+objects = ['table', 'gripper', 'glass', 'vase', 'plastic bottle', 'block', 'phone case']
+# Query: anything fragile.
+fragile_items = []
+for obj in ['glass', 'vase']:
+    item = detect(obj)[0]
+    fragile_items.append(item)
+ret_val = fragile_items
+
+objects = ['table', 'gripper', 'fridge']
+# Query: fridge handle.
+fridge_handle = detect('fridge handle')[0]
+ret_val = fridge_handle
+
+objects = ['table', 'gripper', 'blue block', 'red block']
+# Query: green block.
+ret_val = None
+
+objects = ['table', 'gripper', 'yellow bowl', 'red spoon']
+# Query: gripper.
+gripper = detect('gripper')[0]
+ret_val = gripper

Real/planner/real_planner_prompt.py

@@ -0,0 +1,121 @@
+import numpy as np
+from env_utils import execute
+from perception_utils import parse_query_obj
+import action_utils import composer
+
+objects = ['blue block', 'yellow block', 'mug']
+# Query: place the blue block on the yellow block, and avoid the mug at all time.
+composer("grasp the blue block while keeping at least 15cm away from the mug")
+composer("back to default pose")
+composer("move to 5cm on top of the yellow block while keeping at least 15cm away from the mug")
+composer("open gripper")
+# done
+
+objects = ['airpods', 'drawer']
+# Query: Open the drawer slowly.
+composer("grasp the drawer handle, at 0.5x speed")
+composer("move away from the drawer handle by 25cm, at 0.5x speed")
+composer("open gripper, at 0.5x speed")
+# done
+
+objects = ['tissue box', 'tissue', 'bowl']
+# Query: Can you pass me a tissue and place it next to the bowl?
+composer("grasp the tissue")
+composer("back to default pose")
+composer("move to 10cm to the right of the bowl")
+composer("open gripper")
+composer("back to default pose")
+# done
+
+objects = ['charger', 'outlet']
+# Query: unplug the charger from the wall.
+composer("grasp the charger")
+composer("back to default pose")
+# done
+
+objects = ['grape', 'lemon', 'drill', 'router', 'bread', 'tray']
+# Query: put the sweeter fruit in the tray that contains the bread.
+composer("grasp the grape")
+composer("back to default pose")
+composer("move to the top of the tray that contains the bread")
+composer("open gripper")
+# done
+
+objects = ['marbles', 'tray', 'broom']
+# Query: Can you sweep the marbles into the tray?
+composer("grasp the broom")
+composer("back to default pose")
+composer("push the marbles into the tray")
+# done
+
+objects = ['orange', 'QR code', 'lemon', 'drawer']
+# Query: put the sour fruit into the top drawer.
+composer("grasp the top drawer handle")
+composer("move away from the top drawer handle by 25cm")
+composer("open gripper")
+composer("back to default pose")
+composer("grasp the lemon")
+composer("move to 10cm on top of the top drawer")
+composer("open gripper")
+# done
+
+objects = ['fridge', 'hot soup']
+# Query: Open the fridge door and be careful around the hot soup.
+composer("grasp the fridge handle and keep at least 15cm away from the hot soup")
+composer("move away from the fridge handle by 25cm and keep at least 15cm away from the hot soup")
+composer("open gripper")
+# done
+
+objects = ['cyan bowl', 'yellow bowl', 'box', 'ice cream']
+# Query: move to the top of the cyan bowl.
+composer("move to the top of the cyan bowl")
+# done
+
+objects = ['drawer', 'umbrella']
+# Query: close the drawer.
+composer("push close the drawer handle by 25cm")
+# done
+
+objects = ['plate', 'steak', 'fork', 'knife', 'spoon']
+# Query: Could you please set up the fork for the steak for me?
+composer("grasp the fork")
+composer("back to default pose")
+composer("move to 10cm to the right of the plate")
+composer("open gripper")
+composer("back to default pose")
+# done
+
+objects = ['light switch']
+# Query: Press down the switch.
+composer("close the gripper")
+composer("move to the center of the light switch")
+composer("back to default pose")
+# done
+
+objects = ['beer']
+# Query: turn close the beer.
+composer("grasp the beer cap")
+composer("turn clockwise by 180 degrees")
+composer("back to default pose")
+# done
+
+objects = ['steak', 'grill', 'plate']
+# Query: Take the steak out of the grill and put it flat on the plate.
+composer("grasp the steak")
+composer("back to default pose")
+composer("rotate the gripper to be 45 degrees slanted relative to the plate")
+composer("move to 10cm on top of the plate")
+composer("open gripper")
+composer("back to default pose")
+# done
+
+objects = ['trash can', 'napkin']
+# Query: sort napkin into trash can.
+napkin = parse_query_obj("napkin")
+trash_can = parse_query_obj("trash can")
+while np.linalg.norm(napkin.position - trash_can.position) > 0.1:
+    composer("grasp the napkin")
+    composer("move to 10cm on top of the trash can")
+    composer("open gripper")
+    composer("back to default pose")
+# done

Real/value_maps/real_get_affordance_map_prompt.py

@@ -0,0 +1,146 @@
+import numpy as np
+from perception_utils import parse_query_obj
+from plan_utils import get_empty_affordance_map, set_voxel_by_radius, cm2index
+
+# Query: a point 10cm in front of [10, 15, 60].
+affordance_map = get_empty_affordance_map()
+# 10cm in front of so we add to x-axis
+x = 10 + cm2index(10, 'x')
+y = 15
+z = 60
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point on the right side of the table.
+affordance_map = get_empty_affordance_map()
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# right side so y = max_y
+x = center_x
+y = max_y
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 20cm on top of the container.
+affordance_map = get_empty_affordance_map()
+container = parse_query_obj('container')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = container.aabb
+center_x, center_y, center_z = container.position
+# 20cm on top of so we add to z-axis
+x = center_x
+y = center_y
+z = max_z + cm2index(20, 'z')
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 1cm to the left of the brown block.
+affordance_map = get_empty_affordance_map()
+brown_block = parse_query_obj('brown block')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = brown_block.aabb
+center_x, center_y, center_z = brown_block.position
+# 1cm to the left of so we subtract from y-axis
+x = center_x
+y = min_y - cm2index(1, 'y')
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: anywhere within 20cm of the right most block.
+affordance_map = get_empty_affordance_map()
+right_most_block = parse_query_obj('the right most block')
+set_voxel_by_radius(affordance_map, right_most_block.position, radius_cm=20, value=1)
+
+# Query: a point on the back side of the table.
+affordance_map = get_empty_affordance_map()
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# back side so x = min_x
+x = min_x
+y = center_y
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point on the front right corner of the table.
+affordance_map = get_empty_affordance_map()
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# front right corner so x = max_x and y = max_y
+x = max_x
+y = max_y
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 30cm into the topmost drawer handle.
+affordance_map = get_empty_affordance_map()
+top_handle = parse_query_obj('topmost drawer handle')
+# negative normal because we are moving into the handle.
+moving_dir = -top_handle.normal
+affordance_xyz = top_handle.position + cm2index(30, moving_dir)
+affordance_map[affordance_xyz[0], affordance_xyz[1], affordance_xyz[2]] = 1
+ret_val = affordance_map
+
+# Query: a point 5cm above the blue block.
+affordance_map = get_empty_affordance_map()
+blue_block = parse_query_obj('blue block')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = blue_block.aabb
+center_x, center_y, center_z = blue_block.position
+# 5cm above so we add to z-axis
+x = center_x
+y = center_y
+z = max_z + cm2index(5, 'z')
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 20cm away from the leftmost block.
+affordance_map = get_empty_affordance_map()
+leftmost_block = parse_query_obj('leftmost block')
+# positive normal because we are moving away from the block.
+moving_dir = leftmost_block.normal
+affordance_xyz = leftmost_block.position + cm2index(20, moving_dir)
+affordance_map[affordance_xyz[0], affordance_xyz[1], affordance_xyz[2]] = 1
+ret_val = affordance_map
+
+# Query: a point 4cm to the left of and 10cm on top of the tray that contains the lemon.
+affordance_map = get_empty_affordance_map()
+tray_with_lemon = parse_query_obj('tray that contains the lemon')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = tray_with_lemon.aabb
+center_x, center_y, center_z = tray_with_lemon.position
+# 4cm to the left of so we subtract from y-axis, and 10cm on top of so we add to z-axis
+x = center_x
+y = min_y - cm2index(4, 'y')
+z = max_z + cm2index(10, 'z')
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 10cm to the right of [45 49 66], and 5cm above it.
+affordance_map = get_empty_affordance_map()
+# 10cm to the right of so we add to y-axis, and 5cm above it so we add to z-axis
+x = 45
+y = 49 + cm2index(10, 'y')
+z = 66 + cm2index(5, 'z')
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: the blue circle.
+affordance_map = get_empty_affordance_map()
+blue_circle = parse_query_obj('blue circle')
+affordance_map = blue_circle.occupancy_map
+ret_val = affordance_map
+
+# Query: a point 10cm above and 5cm to the left of the yellow bowl.
+affordance_map = get_empty_affordance_map()
+yellow_bowl = parse_query_obj('yellow bowl')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = yellow_bowl.aabb
+center_x, center_y, center_z = yellow_bowl.position
+# 10cm above so we add to z-axis, and 5cm to the left of so we subtract from y-axis
+x = center_x
+y = min_y - cm2index(5, 'y')
+z = max_z + cm2index(10, 'z')
+affordance_map[x, y, z] = 1
+ret_val = affordance_map

Real/value_maps/real_get_avoidance_map_prompt.py

@@ -0,0 +1,36 @@
+import numpy as np
+from perception_utils import parse_query_obj
+from plan_utils import get_empty_avoidance_map, set_voxel_by_radius, cm2index
+
+# Query: 10cm from the bowl.
+avoidance_map = get_empty_avoidance_map()
+bowl = parse_query_obj('bowl')
+set_voxel_by_radius(avoidance_map, bowl.position, radius_cm=10, value=1)
+ret_val = avoidance_map
+
+# Query: 20cm near the mug.
+avoidance_map = get_empty_avoidance_map()
+mug = parse_query_obj('mug')
+set_voxel_by_radius(avoidance_map, mug.position, radius_cm=20, value=1)
+ret_val = avoidance_map
+
+# Query: 20cm around the mug and 10cm around the bowl.
+avoidance_map = get_empty_avoidance_map()
+mug = parse_query_obj('mug')
+set_voxel_by_radius(avoidance_map, mug.position, radius_cm=20, value=1)
+bowl = parse_query_obj('bowl')
+set_voxel_by_radius(avoidance_map, bowl.position, radius_cm=10, value=1)
+ret_val = avoidance_map
+
+# Query: 10cm from anything fragile.
+avoidance_map = get_empty_avoidance_map()
+fragile_objects = parse_query_obj('anything fragile')
+for obj in fragile_objects:
+    set_voxel_by_radius(avoidance_map, obj.position, radius_cm=10, value=1)
+ret_val = avoidance_map
+
+# Query: 10cm from the blue circle.
+avoidance_map = get_empty_avoidance_map()
+blue_circle = parse_query_obj('blue circle')
+set_voxel_by_radius(avoidance_map, blue_circle.position, radius_cm=10, value=1)
+ret_val = avoidance_map

Real/value_maps/real_get_gripper_map_prompt.py

@@ -0,0 +1,48 @@
+import numpy as np
+from perception_utils import parse_query_obj
+from plan_utils import get_empty_gripper_map, set_voxel_by_radius, cm2index
+
+# Query: open everywhere except 1cm around the green block.
+gripper_map = get_empty_gripper_map()
+# open everywhere
+gripper_map[:, :, :] = 0
+# close when 1cm around the green block
+green_block = parse_query_obj('green block')
+set_voxel_by_radius(gripper_map, green_block.position, radius_cm=1, value=1)
+ret_val = gripper_map
+
+# Query: close everywhere but open when on top of the back left corner of the table.
+gripper_map = get_empty_gripper_map()
+# close everywhere
+gripper_map[:, :, :] = 1
+# open when on top of the back left corner of the table
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# back so x = min_x, left so y = min_y, top so we add to z
+x = min_x
+y = min_y
+z = max_z + cm2index(15, 'z')
+set_voxel_by_radius(gripper_map, (x, y, z), radius_cm=10, value=0)
+ret_val = gripper_map
+
+# Query: always open except when you are on the right side of the table.
+gripper_map = get_empty_gripper_map()
+# always open
+gripper_map[:, :, :] = 0
+# close when you are on the right side of the table
+table = parse_query_obj('table')
+center_x, center_y, center_z = table.position
+# right side so y is greater than center_y
+gripper_map[:, center_y:, :] = 1
+
+# Query: always close except when you are on the back side of the table.
+gripper_map = get_empty_gripper_map()
+# always close
+gripper_map[:, :, :] = 1
+# open when you are on the back side of the table
+table = parse_query_obj('table')
+center_x, center_y, center_z = table.position
+# back side so x is less than center_x
+gripper_map[:center_x, :, :] = 0
+ret_val = gripper_map

Real/value_maps/real_get_rotation_map_prompt.py

@@ -0,0 +1,53 @@
+import numpy as np
+from plan_utils import get_empty_rotation_map, set_voxel_by_radius, cm2index, vec2quat
+from perception_utils import parse_query_obj
+from transforms3d.euler import euler2quat, quat2euler
+from transforms3d.quaternions import qmult, qinverse
+
+# Query: face the support surface of the bowl.
+rotation_map = get_empty_rotation_map()
+bowl = parse_query_obj('bowl')
+target_rotation = vec2quat(-bowl.normal)
+rotation_map[:, :, :] = target_rotation
+ret_val = rotation_map
+
+# Query: face the table when within 30cm from table center.
+rotation_map = get_empty_rotation_map()
+table = parse_query_obj('table')
+table_center = table.position
+target_rotation = vec2quat(-table.normal)
+set_voxel_by_radius(rotation_map, table_center, radius_cm=30, value=target_rotation)
+ret_val = rotation_map
+
+# Query: face the blue bowl.
+rotation_map = get_empty_rotation_map()
+blue_bowl = parse_query_obj('brown block')
+target_rotation = vec2quat(-blue_bowl.normal)
+rotation_map[:, :, :] = target_rotation
+ret_val = rotation_map
+
+# Query: turn clockwise by 45 degrees when at the center of the beer cap.
+rotation_map = get_empty_rotation_map()
+beer_cap = parse_query_obj('beer cap')
+(x, y, z) = beer_cap.position
+curr_rotation = rotation_map[x, y, z]
+rotation_delta = euler2quat(0, 0, np.pi / 4)
+rotation_map[x, y, z] = qmult(curr_rotation, rotation_delta)
+ret_val = rotation_map
+
+# Query: turn counter-clockwise by 30 degrees.
+rotation_map = get_empty_rotation_map()
+curr_rotation = rotation_map[0, 0, 0]
+rotation_delta = euler2quat(0, 0, -np.pi / 6)
+rotation_map[:, :, :] = qmult(curr_rotation, rotation_delta)
+ret_val = rotation_map
+
+# Query: rotate the gripper to be 45 degrees slanted relative to the plate.
+rotation_map = get_empty_rotation_map()
+plate = parse_query_obj('plate')
+face_plate_quat = vec2quat(-plate.normal)
+# rotate 45 degrees around the x-axis
+rotation_delta = euler2quat(-np.pi / 4, 0, 0)
+target_rotation = qmult(face_plate_quat, rotation_delta)
+rotation_map[:, :, :] = target_rotation
+ret_val = rotation_map

Real/value_maps/real_get_velocity_map_prompt.py

@@ -0,0 +1,31 @@
+import numpy as np
+from plan_utils import get_empty_velocity_map, set_voxel_by_radius, cm2index
+from perception_utils import parse_query_obj
+
+# Query: faster when on the right side of the table and slower when on the left side of the table.
+velocity_map = get_empty_velocity_map()
+table = parse_query_obj('table')
+center_x, center_y, center_z = table.position
+# faster on right side so 1.5 when y > center_y, slower on left side so 0.5 when y < center_y
+velocity_map[:, center_y:, :] = 1.5
+velocity_map[:, :center_y, :] = 0.5
+ret_val = velocity_map
+
+# Query: slow down by a quarter.
+velocity_map = get_empty_velocity_map()
+velocity_map[:] = 0.75
+ret_val = velocity_map
+
+# Query: slow down by a half when you're near anything fragile (objects: ['block', 'fork', 'mug', 'bowl', 'chips']).
+velocity_map = get_empty_velocity_map()
+mug = parse_query_obj('mug')
+set_voxel_by_radius(velocity_map, mug.position, radius_cm=10, value=0.5)
+bowl = parse_query_obj('bowl')
+set_voxel_by_radius(velocity_map, bowl.position, radius_cm=10, value=0.5)
+ret_val = velocity_map
+
+# Query: quarter of the speed when within 9cm from the yellow line.
+velocity_map = get_empty_velocity_map()
+yellow_line = parse_query_obj('yellow_line')
+set_voxel_by_radius(velocity_map, yellow_line.position, radius_cm=9, value=0.25)
+ret_val = velocity_map

Sim/composer/sim_composer_prompt.py

@@ -0,0 +1,91 @@
+import numpy as np
+from env_utils import execute, reset_to_default_pose
+from perception_utils import parse_query_obj
+from plan_utils import get_affordance_map, get_avoidance_map, get_velocity_map, get_rotation_map, get_gripper_map
+
+# Query: move ee forward for 7cm.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map(f'a point 7cm in front of {movable.position}')
+execute(movable, affordance_map)
+
+# Query: move to the left of the green block.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point 11cm to the left of the green block')
+execute(movable, affordance_map=affordance_map)
+
+# Query: move to the yellow block while staying on the front side the pink block.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point 11cm on top of the yellow block')
+avoidance_map = get_avoidance_map('anywhere behind the pink block')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map)
+
+# Query: move to the back side of the table.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point on the back side of the table')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map)
+
+# Query: move to the front right corner of the table while moving at faster speed when within 7cm from the yellow block.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point on the front right corner of the table')
+velocity_map = get_velocity_map('faster speed when within 7cm from the yellow block')
+execute(movable, affordance_map=affordance_map, velocity_map=velocity_map)
+
+# Query: move to 15cm on top of the rightmost block while avoiding the blue line and moving at a quarter of the speed on the right side of the table.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point 15cm on top of the rightmost block')
+avoidance_map = get_avoidance_map('the blue line')
+velocity_map = get_velocity_map('a quarter of the speed on the right side of the table')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map, velocity_map=velocity_map)
+
+# Query: close the topmost drawer by pushing.
+movable = parse_query_obj('topmost drawer handle')
+affordance_map = get_affordance_map('a point 21cm into the topmost drawer handle')
+execute(movable, affordance_map=affordance_map)
+
+# Query: push the second to the left block along the brown line.
+movable = parse_query_obj('second to the left block')
+affordance_map = get_affordance_map('the brown line')
+execute(movable, affordance_map=affordance_map)
+
+# Query: grasp the blue block from the table at a quarter of the speed.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point at the center of blue block')
+rotation_map = get_rotation_map('face the blue block')
+velocity_map = get_velocity_map('quarter of the speed')
+gripper_map = get_gripper_map('open everywhere except 1cm around the blue block')
+execute(movable, affordance_map=affordance_map, rotation_map=rotation_map, velocity_map=velocity_map, gripper_map=gripper_map)
+
+# Query: move to the left of the brown block.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point 11cm to the left of the brown block')
+execute(movable, affordance_map=affordance_map)
+
+# Query: back to default pose.
+reset_to_default_pose()
+
+# Query: open gripper.
+movable = parse_query_obj('gripper')
+gripper_map = get_gripper_map('open everywhere')
+execute(movable, gripper_map=gripper_map)
+
+# Query: drop the blue block to the right side of the table while staying at least 7cm away from the brown block.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point on the right side of the table')
+avoidance_map = get_avoidance_map('7cm away from the brown block')
+gripper_map = get_gripper_map('close everywhere except 1cm around the right side of the table')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map, gripper_map=gripper_map)
+
+# Query: move to the front side of the yellow block.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point 11cm in front of the yellow block')
+execute(movable, affordance_map=affordance_map)
+
+# Query: move to the back side of the table.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point on the back side of the table')
+execute(movable, affordance_map=affordance_map)
+
+# Query: stay away from the brown block.
+movable = parse_query_obj('ee')
+avoidance_map = get_avoidance_map('3cm away from the brown block')
+execute(movable, avoidance_map=avoidance_map)

Sim/perception/sim_parse_query_obj_prompt.py

@@ -0,0 +1,71 @@
+import numpy as np
+from perception_utils import detect
+
+objects = ['green block', 'yellow line']
+# Query: ee.
+ee = detect('ee')[0]
+ret_val = ee
+
+objects = ['drawer', 'blue block', 'yellow block']
+# Query: topmost handle.
+handles = detect('drawer handle')
+# topmost so sort by z, take the last one
+handles = sorted(handles, key=lambda x: x.position[2])
+top_handle = handles[-1]
+ret_val = top_handle
+
+objects = ['yellow block', 'pink block', 'brown line', 'pink block']
+# Query: second to the left block.
+blocks = detect('block')
+# second to the left so sort by y, take the second one
+blocks = sorted(blocks, key=lambda x: x.position[1])
+second_left_block = blocks[1]
+ret_val = second_left_block
+
+objects = ['blue line', 'pink line', 'green block']
+# Query: table.
+table = detect('table')[0]
+ret_val = table
+
+objects = ['green line', 'drawer', 'yellow block']
+# Query: second to the bottom handle.
+handles = detect('drawer handle')
+# second to the bottom so sort by z, take the second one
+handles = sorted(handles, key=lambda x: x.position[2])
+second_bottom_handle = handles[1]
+ret_val = second_bottom_handle
+
+objects = ['brown line', 'brown block']
+# Query: brown line.
+brown_line = detect('brown line')[0]
+ret_val = brown_line
+
+objects = ['green block', 'brown block', 'yellow line']
+# Query: block.
+block = detect('green block')[0]
+ret_val = block
+
+objects = ['pink block', 'pink line', 'blue block']
+# Query: block closest to the pink line.
+blocks = detect('block')
+pink_line = detect('pink line')[0]
+closest_block = min(blocks, key=lambda x: np.linalg.norm(x.position - pink_line.position))
+ret_val = closest_block
+
+objects = ['blue block', 'blue line', 'green block', 'pink block', 'brown block']
+# Query: the block that is on top of the blue block.
+blocks = detect('block')
+blue_block = detect('blue block')[0]
+# find the block that is on top of the blue block
+for block in blocks:
+    if block.position[2] > blue_block.position[2]:
+        ret_val = block
+        break
+
+objects = ['drawer']
+# Query: top drawer handle.
+handles = detect('drawer handle')
+# top drawer handle so sort by z, take the last one
+handles = sorted(handles, key=lambda x: x.position[2])
+top_handle = handles[-1]
+ret_val = top_handle

Sim/value_maps/sim_get_affordance_map_prompt.py

@@ -0,0 +1,177 @@
+import numpy as np
+from perception_utils import parse_query_obj
+from plan_utils import get_empty_affordance_map, set_voxel_by_radius, cm2index
+
+# Query: a point 10cm in front of [10, 15, 60].
+affordance_map = get_empty_affordance_map()
+# in front so we add to x
+x = 10 + cm2index(10, 'x')
+y = 15
+z = 60
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point on the back side of the table.
+affordance_map = get_empty_affordance_map()
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# back side so x = min_x
+x = min_x
+y = center_y
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 21cm on top of the green block.
+affordance_map = get_empty_affordance_map()
+green_block = parse_query_obj('green block')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = green_block.aabb
+center_x, center_y, center_z = green_block.position
+# 21cm on top so z = max_z + 21cm
+x = center_x
+y = center_y
+z = max_z + cm2index(21, 'z')
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 1cm to the left of the brown block.
+affordance_map = get_empty_affordance_map()
+brown_block = parse_query_obj('brown block')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = brown_block.aabb
+center_x, center_y, center_z = brown_block.position
+# 1cm to the left so y = min_y - 1cm
+x = center_x
+y = min_y - cm2index(1, 'y')
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: anywhere within 15cm of the right most block.
+affordance_map = get_empty_affordance_map()
+right_most_block = parse_query_obj('the right most block')
+set_voxel_by_radius(affordance_map, right_most_block.position, radius_cm=15, value=1)
+ret_val = affordance_map
+
+# Query: a point on the back side of the table.
+affordance_map = get_empty_affordance_map()
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# back side so x = min_x
+x = min_x
+y = center_y
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point on the front right corner of the table.
+affordance_map = get_empty_affordance_map()
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# front right corner -> front so x = max_x, right so y = max_y
+x = max_x
+y = max_y
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 30cm into the topmost drawer handle.
+affordance_map = get_empty_affordance_map()
+top_handle = parse_query_obj('topmost drawer handle')
+# negative normal because we are moving into the handle.
+moving_dir = -top_handle.normal
+affordance_xyz = top_handle.position + cm2index(30, moving_dir)
+affordance_map[affordance_xyz[0], affordance_xyz[1], affordance_xyz[2]] = 1
+ret_val = affordance_map
+
+# Query: a point 9cm to the left of the blue block.
+affordance_map = get_empty_affordance_map()
+blue_block = parse_query_obj('blue block')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = blue_block.aabb
+center_x, center_y, center_z = blue_block.position
+# 9cm to the left so y = min_y - 9cm
+x = center_x
+y = min_y - cm2index(9, 'y')
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 21cm away from the leftmost block.
+affordance_map = get_empty_affordance_map()
+leftmost_block = parse_query_obj('leftmost block')
+# positive normal because we are moving away from the block.
+moving_dir = leftmost_block.normal
+affordance_xyz = leftmost_block.position + cm2index(21, moving_dir)
+affordance_map[affordance_xyz[0], affordance_xyz[1], affordance_xyz[2]] = 1
+ret_val = affordance_map
+
+# Query: a point 7cm above the tray that contains the lemon.
+affordance_map = get_empty_affordance_map()
+tray_with_lemon = parse_query_obj('tray that contains the lemon')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = tray_with_lemon.aabb
+center_x, center_y, center_z = tray_with_lemon.position
+# 7cm above so z = max_z + 7cm
+x = center_x
+y = center_y
+z = max_z + cm2index(7, 'z')
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 3cm to the left of [45, 49, 99].
+affordance_map = get_empty_affordance_map()
+# left so y = 49 - 3cm
+x = 45
+y = 49 - cm2index(3, 'y')
+z = 99
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: the brown line.
+brown_line = parse_query_obj('brown line')
+affordance_map = brown_line.occupancy_map
+ret_val = affordance_map
+
+# Query: a point 13cm to the left of the gripper.
+affordance_map = get_empty_affordance_map()
+gripper = parse_query_obj('gripper')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = gripper.aabb
+center_x, center_y, center_z = gripper.position
+# 13cm to the left so y = min_y - 13cm
+x = center_x
+y = min_y - cm2index(13, 'y')
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 17cm above the back left corner of the table.
+affordance_map = get_empty_affordance_map()
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# 17cm on top of back left corner so x = min_x, y = min_y, z = max_z + 17cm
+x = min_x
+y = min_y
+z = max_z + cm2index(17, 'z')
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point at the center of the green block.
+affordance_map = get_empty_affordance_map()
+green_block = parse_query_obj('green block')
+center_x, center_y, center_z = green_block.position
+affordance_map[center_x, center_y, center_z] = 1
+ret_val = affordance_map
+
+# Query: the right side of the table.
+affordance_map = get_empty_affordance_map()
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# right side so y = max_y
+x = center_x
+y = max_y
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map

Sim/value_maps/sim_get_avoidance_map_prompt.py

@@ -0,0 +1,60 @@
+import numpy as np
+from perception_utils import parse_query_obj
+from plan_utils import get_empty_avoidance_map, set_voxel_by_radius, cm2index
+
+# Query: 11cm from the blue block.
+avoidance_map = get_empty_avoidance_map()
+blue_block = parse_query_obj('blue block')
+set_voxel_by_radius(avoidance_map, blue_block.position, radius_cm=11, value=1)
+ret_val = avoidance_map
+
+# Query: 7cm near the pink block.
+avoidance_map = get_empty_avoidance_map()
+pink_block = parse_query_obj('pink block')
+set_voxel_by_radius(avoidance_map, pink_block.position, radius_cm=7, value=1)
+ret_val = avoidance_map
+
+# Query: 13cm around the brown block and 5cm around the green block.
+avoidance_map = get_empty_avoidance_map()
+brown_block = parse_query_obj('brown block')
+set_voxel_by_radius(avoidance_map, brown_block.position, radius_cm=13, value=1)
+green_block = parse_query_obj('green block')
+set_voxel_by_radius(avoidance_map, green_block.position, radius_cm=5, value=1)
+ret_val = avoidance_map
+
+# Query: the blue line.
+blue_line = parse_query_obj('blue_line')
+avoidance_map = blue_line.occupancy_map
+ret_val = avoidance_map
+
+# Query: anywhere on the front side of the blue block.
+avoidance_map = get_empty_avoidance_map()
+blue_block = parse_query_obj('blue block')
+center_x, center_y, center_z = blue_block.position
+# front side so x > center_x
+avoidance_map[center_x:, :, :] = 1
+ret_val = avoidance_map
+
+# Query: anywhere on the left of the green block.
+avoidance_map = get_empty_avoidance_map()
+green_block = parse_query_obj('green block')
+center_x, center_y, center_z = green_block.position
+# left side so y < center_y
+avoidance_map[:, :center_y, :] = 1
+ret_val = avoidance_map
+
+# Query: anywhere behind the pink block.
+avoidance_map = get_empty_avoidance_map()
+pink_block = parse_query_obj('pink block')
+center_x, center_y, center_z = pink_block.position
+# behind so x < center_x
+avoidance_map[:center_x, :, :] = 1
+ret_val = avoidance_map
+
+# Query: anywhere above the brown block.
+avoidance_map = get_empty_avoidance_map()
+brown_block = parse_query_obj('brown block')
+center_x, center_y, center_z = brown_block.position
+# above so z > center_z
+avoidance_map[:, :, center_z:] = 1
+ret_val = avoidance_map

Sim/value_maps/sim_get_gripper_map_prompt.py

@@ -0,0 +1,57 @@
+import numpy as np
+from perception_utils import parse_query_obj
+from plan_utils import get_empty_gripper_map, set_voxel_by_radius, cm2index
+
+# Query: open everywhere except 1cm around the green block.
+gripper_map = get_empty_gripper_map()
+# open everywhere
+gripper_map[:, :, :] = 0
+# close when 1cm around the green block
+green_block = parse_query_obj('green block')
+set_voxel_by_radius(gripper_map, green_block.position, radius_cm=1, value=1)
+ret_val = gripper_map
+
+# Query: close everywhere but open when on top of the back left corner of the table.
+gripper_map = get_empty_gripper_map()
+# close everywhere
+gripper_map[:, :, :] = 1
+# open when on top of the back left corner of the table
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# back so x = min_x, left so y = min_y, top so we add to z
+x = min_x
+y = min_y
+z = max_z + cm2index(10, 'z')
+set_voxel_by_radius(gripper_map, (x, y, z), radius_cm=10, value=0)
+ret_val = gripper_map
+
+# Query: always open except when you are on the right side of the table.
+gripper_map = get_empty_gripper_map()
+# always open
+gripper_map[:, :, :] = 0
+# close when you are on the right side of the table
+table = parse_query_obj('table')
+center_x, center_y, center_z = table.position
+# right side so y is greater than center_y
+gripper_map[:, center_y:, :] = 1
+
+# Query: always close except when you are on the back side of the table.
+gripper_map = get_empty_gripper_map()
+# always close
+gripper_map[:, :, :] = 1
+# open when you are on the back side of the table
+table = parse_query_obj('table')
+center_x, center_y, center_z = table.position
+# back side so x is less than center_x
+gripper_map[:center_x, :, :] = 0
+ret_val = gripper_map
+
+# Query: open everywhere except 1cm around the topmost drawer handle.
+gripper_map = get_empty_gripper_map()
+# open everywhere
+gripper_map[:, :, :] = 0
+# close when 1cm around the topmost drawer handle
+topmost_drawer_handle = parse_query_obj('topmost drawer handle')
+set_voxel_by_radius(gripper_map, topmost_drawer_handle.position, radius_cm=1, value=1)
+ret_val = gripper_map

Sim/value_maps/sim_get_rotation_map_prompt.py

@@ -0,0 +1,25 @@
+import numpy as np
+from plan_utils import get_empty_rotation_map, set_voxel_by_radius, cm2index, vec2quat
+from perception_utils import parse_query_obj
+
+# Query: face the green block.
+rotation_map = get_empty_rotation_map()
+green_block = parse_query_obj('green block')
+target_rotation = vec2quat(-green_block.normal)
+rotation_map[:, :, :] = target_rotation
+ret_val = rotation_map
+
+# Query: face the table when within 30cm from table center.
+rotation_map = get_empty_rotation_map()
+table = parse_query_obj('table')
+table_center = table.position
+target_rotation = vec2quat(-table.normal)
+set_voxel_by_radius(rotation_map, table_center, radius_cm=30, value=target_rotation)
+ret_val = rotation_map
+
+# Query: face the topmost drawer handle.
+rotation_map = get_empty_rotation_map()
+topmost_drawer_handle = parse_query_obj('topmost drawer handle')
+target_rotation = vec2quat(-topmost_drawer_handle.normal)
+rotation_map[:, :, :] = target_rotation
+ret_val = rotation_map

Sim/value_maps/sim_get_velocity_map_prompt.py

@@ -0,0 +1,46 @@
+import numpy as np
+from plan_utils import get_empty_velocity_map, set_voxel_by_radius, cm2index
+from perception_utils import parse_query_obj
+
+# Query: faster when on the left side of the table and a quarter of the speed when on the other side of the table.
+velocity_map = get_empty_velocity_map()
+table = parse_query_obj('table')
+center_x, center_y, center_z = table.position
+# faster on left side so y < center_y
+velocity_map[:, :center_y, :] = 1.5
+# a quarter of the speed on the other side so y > center_y
+velocity_map[:, center_y:, :] = 0.25
+ret_val = velocity_map
+
+# Query: slow down by a quarter.
+velocity_map = get_empty_velocity_map()
+velocity_map[:] = 0.75
+ret_val = velocity_map
+
+# Query: quarter of the speed when within 9cm from the yellow block.
+velocity_map = get_empty_velocity_map()
+yellow_block = parse_query_obj('yellow block')
+set_voxel_by_radius(velocity_map, yellow_block.position, radius_cm=9, value=0.25)
+ret_val = velocity_map
+
+# Query: quarter of the speed in the back side of the table.
+velocity_map = get_empty_velocity_map()
+table = parse_query_obj('table')
+center_x, center_y, center_z = table.position
+# a quarter of the speed in the back side so x < center_x
+velocity_map[:center_x, :, :] = 0.25
+ret_val = velocity_map
+
+# Query: faster speed in the right side of the table.
+velocity_map = get_empty_velocity_map()
+table = parse_query_obj('table')
+center_x, center_y, center_z = table.position
+# faster in the right side so y > center_y
+velocity_map[:, center_y:, :] = 1.5
+ret_val = velocity_map
+
+# Query: quarter of the speed when within 11cm from the brown block.
+velocity_map = get_empty_velocity_map()
+brown_block = parse_query_obj('brown_block')
+set_voxel_by_radius(velocity_map, brown_block.position, radius_cm=11, value=0.25)
+ret_val = velocity_map

VoxPoserApiExamples/Real/composer/real_composer_prompt.py

@@ -0,0 +1,120 @@
+import numpy as np
+from env_utils import execute, reset_to_default_pose
+from perception_utils import parse_query_obj
+from plan_utils import get_affordance_map, get_avoidance_map, get_velocity_map, get_rotation_map, get_gripper_map
+
+# Query: move ee forward for 10cm.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map(f'a point 10cm in front of {movable.position}')
+execute(movable, affordance_map)
+
+# Query: go back to default.
+reset_to_default_pose()
+
+# Query: move the gripper behind the bowl, and slow down when near the bowl.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point 15cm behind the bowl')
+avoidance_map = get_avoidance_map('10cm near the bowl')
+velocity_map = get_velocity_map('slow down when near the bowl')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map, velocity_map=velocity_map)
+
+# Query: move to the back side of the table while staying at least 5cm from the blue block.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point on the back side of the table')
+avoidance_map = get_avoidance_map('5cm from the blue block')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map)
+
+# Query: move to the top of the plate and face the plate.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point 10cm above the plate')
+rotation_map = get_rotation_map('face the plate')
+execute(movable, affordance_map=affordance_map, rotation_map=rotation_map)
+
+# Query: drop the toy inside container.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point 15cm above the container')
+gripper_map = get_gripper_map('close everywhere but open when on top of the container')
+execute(movable, affordance_map=affordance_map, gripper_map=gripper_map)
+
+# Query: push close the topmost drawer.
+movable = parse_query_obj('topmost drawer handle')
+affordance_map = get_affordance_map('a point 30cm into the topmost drawer handle')
+execute(movable, affordance_map=affordance_map)
+
+# Query: push the second to the left block along the red line.
+movable = parse_query_obj('second to the left block')
+affordance_map = get_affordance_map('the red line')
+execute(movable, affordance_map=affordance_map)
+
+# Query: grasp the blue block from the table at a quarter of the speed.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point at the center of blue block')
+velocity_map = get_velocity_map('quarter of the speed')
+gripper_map = get_gripper_map('open everywhere except 1cm around the blue block')
+execute(movable, affordance_map=affordance_map, velocity_map=velocity_map, gripper_map=gripper_map)
+
+# Query: move to the left of the brown block.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point 10cm to the left of the brown block')
+execute(movable, affordance_map=affordance_map)
+
+# Query: move to the top of the tray that contains the lemon.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point 10cm above the tray that contains the lemon')
+execute(movable, affordance_map=affordance_map)
+
+# Query: close drawer by 5cm.
+movable = parse_query_obj('drawer handle')
+affordance_map = get_affordance_map('a point 5cm into the drawer handle')
+execute(movable, affordance_map=affordance_map)
+
+# Query: move to 5cm on top of the soda can, at 0.5x speed when within 20cm of the wooden mug, and keep at least 15cm away from the wooden mug.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point 5cm above the soda can')
+avoidance_map = get_avoidance_map('15cm from the wooden mug')
+velocity_map = get_velocity_map('0.5x speed when within 20cm of the wooden mug')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map, velocity_map=velocity_map)
+
+# Query: wipe the red dot but avoid the blue block.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('the red dot')
+avoidance_map = get_avoidance_map('10cm from the blue block')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map)
+
+# Query: grasp the mug from the shelf.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point at the center of the mug handle')
+gripper_map = get_gripper_map('open everywhere except 1cm around the mug handle')
+execute(movable, affordance_map=affordance_map, gripper_map=gripper_map)
+
+# Query: move to 10cm on top of the soup bowl, and 5cm to the left of the soup bowl, while away from the glass, at 0.75x speed.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point 10cm above and 5cm to the left of the soup bowl')
+avoidance_map = get_avoidance_map('10cm from the glass')
+velocity_map = get_velocity_map('0.75x speed')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map, velocity_map=velocity_map)
+
+# Query: open gripper.
+movable = parse_query_obj('gripper')
+gripper_map = get_gripper_map('open everywhere')
+execute(movable, gripper_map=gripper_map)
+
+# Query: turn counter-clockwise by 180 degrees.
+movable = parse_query_obj('gripper')
+rotation_map = get_rotation_map('turn counter-clockwise by 180 degrees')
+execute(movable, rotation_map=rotation_map)
+
+# Query: sweep all particles to the left side of the table.
+particles = parse_query_obj('particles')
+for particle in particles:
+    movable = particle
+    affordance_map = get_affordance_map('a point on the left side of the table')
+    execute(particle, affordance_map=affordance_map)
+
+# Query: grasp the bottom drawer handle while moving at 0.5x speed.
+movable = parse_query_obj('gripper')
+affordance_map = get_affordance_map('a point at the center of the bottom drawer handle')
+velocity_map = get_velocity_map('0.5x speed')
+rotation_map = get_rotation_map('face the bottom drawer handle')
+gripper_map = get_gripper_map('open everywhere except 1cm around the bottom drawer handle')
+execute(movable, affordance_map=affordance_map, velocity_map=velocity_map, rotation_map=rotation_map, gripper_map=gripper_map)

VoxPoserApiExamples/Real/perception/real_parse_query_obj_prompt.py

@@ -0,0 +1,104 @@
+import numpy as np
+from perception_utils import detect
+
+objects = ['table', 'gripper', 'green block', 'cardboard box']
+# Query: gripper.
+gripper = detect('gripper')[0]
+ret_val = gripper
+
+objects = ['table', 'gripper', 'drawer', 'egg', 'egg', 'plate']
+# Query: topmost handle.
+handles = detect('drawer handle')
+handles = sorted(handles, key=lambda x: x.position[2])
+top_handle = handles[-1]
+ret_val = top_handle
+
+objects = ['table', 'gripper', 'yellow block', 'charging cable', 'cyan block', 'magenta block']
+# Query: second to the left block.
+blocks = detect('block')
+blocks = sorted(blocks, key=lambda x: x.position[1])
+second_left_block = blocks[1]
+ret_val = second_left_block
+
+objects = ['table', 'gripper', 'iPhone', 'ruler', 'pink line', 'blue line']
+# Query: the front most line on the table.
+lines = detect('line')
+lines = sorted(lines, key=lambda x: x.position[0])
+front_most_line = lines[-1]
+ret_val = front_most_line
+
+objects = ['table', 'gripper', 'vase', 'napkin box', 'mask']
+# Query: table.
+table = detect('table')[0]
+ret_val = table
+
+objects = ['table', 'gripper', 'bottle', 'drawer', 'bowl', 'bag']
+# Query: second to the bottom handle.
+handles = detect('drawer handle')
+handles = sorted(handles, key=lambda x: x.position[2])
+second_bottom_handle = handles[1]
+ret_val = second_bottom_handle
+
+objects = ['table', 'gripper', 'brown line', 'red block', 'monitor']
+# Query: brown line.
+brown_line = detect('brown line')[0]
+ret_val = brown_line
+
+objects = ['table', 'gripper', 'green block', 'cup holder', 'black block']
+# Query: block.
+block = detect('green block')[0]
+ret_val = block
+
+objects = ['table', 'gripper', 'mouse', 'yellow bowl', 'brown bowl', 'sticker']
+# Query: bowl closest to the sticker.
+bowls = detect('bowl')
+sticker = detect('sticker')[0]
+closest_bowl = min(bowls, key=lambda x: np.linalg.norm(x.position - sticker.position))
+ret_val = closest_bowl
+
+objects = ['table', 'gripper', 'keyboard', 'brown bag', 'pink bag', 'red tape', 'bottle']
+# Query: bag with the red tape on top.
+bags = detect('bag')
+red_tape = detect('red tape')[0]
+bag_with_red_tape = min(bags, key=lambda x: np.linalg.norm(x.position - red_tape.position))
+ret_val = bag_with_red_tape
+
+objects = ['table', 'gripper', 'grape', 'wood tray', 'strawberry', 'white tray', 'blue tray', 'bread']
+# Query: tray that contains the bread.
+trays = detect('tray')
+bread = detect('bread')[0]
+tray_with_bread = min(trays, key=lambda x: np.linalg.norm(x.position - bread.position))
+ret_val = tray_with_bread
+
+objects = ['table', 'gripper', 'drawer']
+# Query: top drawer handle.
+handles = detect('drawer handle')
+top_drawer_handle = max(handles, key=lambda x: x.position[2])
+ret_val = top_drawer_handle
+
+objects = ['table', 'gripper', 'door']
+# Query: the thing you can open the door with.
+door_handle = detect('door handle')[0]
+ret_val = door_handle
+
+objects = ['table', 'gripper', 'glass', 'vase', 'plastic bottle', 'block', 'phone case']
+# Query: anything fragile.
+fragile_items = []
+for obj in ['glass', 'vase']:
+    item = detect(obj)[0]
+    fragile_items.append(item)
+ret_val = fragile_items
+
+objects = ['table', 'gripper', 'fridge']
+# Query: fridge handle.
+fridge_handle = detect('fridge handle')[0]
+ret_val = fridge_handle
+
+objects = ['table', 'gripper', 'blue block', 'red block']
+# Query: green block.
+ret_val = None
+
+objects = ['table', 'gripper', 'yellow bowl', 'red spoon']
+# Query: gripper.
+gripper = detect('gripper')[0]
+ret_val = gripper

VoxPoserApiExamples/Real/planner/real_planner_prompt.py

@@ -0,0 +1,121 @@
+import numpy as np
+from env_utils import execute
+from perception_utils import parse_query_obj
+import action_utils import composer
+
+objects = ['blue block', 'yellow block', 'mug']
+# Query: place the blue block on the yellow block, and avoid the mug at all time.
+composer("grasp the blue block while keeping at least 15cm away from the mug")
+composer("back to default pose")
+composer("move to 5cm on top of the yellow block while keeping at least 15cm away from the mug")
+composer("open gripper")
+# done
+
+objects = ['airpods', 'drawer']
+# Query: Open the drawer slowly.
+composer("grasp the drawer handle, at 0.5x speed")
+composer("move away from the drawer handle by 25cm, at 0.5x speed")
+composer("open gripper, at 0.5x speed")
+# done
+
+objects = ['tissue box', 'tissue', 'bowl']
+# Query: Can you pass me a tissue and place it next to the bowl?
+composer("grasp the tissue")
+composer("back to default pose")
+composer("move to 10cm to the right of the bowl")
+composer("open gripper")
+composer("back to default pose")
+# done
+
+objects = ['charger', 'outlet']
+# Query: unplug the charger from the wall.
+composer("grasp the charger")
+composer("back to default pose")
+# done
+
+objects = ['grape', 'lemon', 'drill', 'router', 'bread', 'tray']
+# Query: put the sweeter fruit in the tray that contains the bread.
+composer("grasp the grape")
+composer("back to default pose")
+composer("move to the top of the tray that contains the bread")
+composer("open gripper")
+# done
+
+objects = ['marbles', 'tray', 'broom']
+# Query: Can you sweep the marbles into the tray?
+composer("grasp the broom")
+composer("back to default pose")
+composer("push the marbles into the tray")
+# done
+
+objects = ['orange', 'QR code', 'lemon', 'drawer']
+# Query: put the sour fruit into the top drawer.
+composer("grasp the top drawer handle")
+composer("move away from the top drawer handle by 25cm")
+composer("open gripper")
+composer("back to default pose")
+composer("grasp the lemon")
+composer("move to 10cm on top of the top drawer")
+composer("open gripper")
+# done
+
+objects = ['fridge', 'hot soup']
+# Query: Open the fridge door and be careful around the hot soup.
+composer("grasp the fridge handle and keep at least 15cm away from the hot soup")
+composer("move away from the fridge handle by 25cm and keep at least 15cm away from the hot soup")
+composer("open gripper")
+# done
+
+objects = ['cyan bowl', 'yellow bowl', 'box', 'ice cream']
+# Query: move to the top of the cyan bowl.
+composer("move to the top of the cyan bowl")
+# done
+
+objects = ['drawer', 'umbrella']
+# Query: close the drawer.
+composer("push close the drawer handle by 25cm")
+# done
+
+objects = ['plate', 'steak', 'fork', 'knife', 'spoon']
+# Query: Could you please set up the fork for the steak for me?
+composer("grasp the fork")
+composer("back to default pose")
+composer("move to 10cm to the right of the plate")
+composer("open gripper")
+composer("back to default pose")
+# done
+
+objects = ['light switch']
+# Query: Press down the switch.
+composer("close the gripper")
+composer("move to the center of the light switch")
+composer("back to default pose")
+# done
+
+objects = ['beer']
+# Query: turn close the beer.
+composer("grasp the beer cap")
+composer("turn clockwise by 180 degrees")
+composer("back to default pose")
+# done
+
+objects = ['steak', 'grill', 'plate']
+# Query: Take the steak out of the grill and put it flat on the plate.
+composer("grasp the steak")
+composer("back to default pose")
+composer("rotate the gripper to be 45 degrees slanted relative to the plate")
+composer("move to 10cm on top of the plate")
+composer("open gripper")
+composer("back to default pose")
+# done
+
+objects = ['trash can', 'napkin']
+# Query: sort napkin into trash can.
+napkin = parse_query_obj("napkin")
+trash_can = parse_query_obj("trash can")
+while np.linalg.norm(napkin.position - trash_can.position) > 0.1:
+    composer("grasp the napkin")
+    composer("move to 10cm on top of the trash can")
+    composer("open gripper")
+    composer("back to default pose")
+# done

VoxPoserApiExamples/Real/value_maps/real_get_affordance_map_prompt.py

@@ -0,0 +1,146 @@
+import numpy as np
+from perception_utils import parse_query_obj
+from plan_utils import get_empty_affordance_map, set_voxel_by_radius, cm2index
+
+# Query: a point 10cm in front of [10, 15, 60].
+affordance_map = get_empty_affordance_map()
+# 10cm in front of so we add to x-axis
+x = 10 + cm2index(10, 'x')
+y = 15
+z = 60
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point on the right side of the table.
+affordance_map = get_empty_affordance_map()
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# right side so y = max_y
+x = center_x
+y = max_y
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 20cm on top of the container.
+affordance_map = get_empty_affordance_map()
+container = parse_query_obj('container')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = container.aabb
+center_x, center_y, center_z = container.position
+# 20cm on top of so we add to z-axis
+x = center_x
+y = center_y
+z = max_z + cm2index(20, 'z')
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 1cm to the left of the brown block.
+affordance_map = get_empty_affordance_map()
+brown_block = parse_query_obj('brown block')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = brown_block.aabb
+center_x, center_y, center_z = brown_block.position
+# 1cm to the left of so we subtract from y-axis
+x = center_x
+y = min_y - cm2index(1, 'y')
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: anywhere within 20cm of the right most block.
+affordance_map = get_empty_affordance_map()
+right_most_block = parse_query_obj('the right most block')
+set_voxel_by_radius(affordance_map, right_most_block.position, radius_cm=20, value=1)
+
+# Query: a point on the back side of the table.
+affordance_map = get_empty_affordance_map()
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# back side so x = min_x
+x = min_x
+y = center_y
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point on the front right corner of the table.
+affordance_map = get_empty_affordance_map()
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# front right corner so x = max_x and y = max_y
+x = max_x
+y = max_y
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 30cm into the topmost drawer handle.
+affordance_map = get_empty_affordance_map()
+top_handle = parse_query_obj('topmost drawer handle')
+# negative normal because we are moving into the handle.
+moving_dir = -top_handle.normal
+affordance_xyz = top_handle.position + cm2index(30, moving_dir)
+affordance_map[affordance_xyz[0], affordance_xyz[1], affordance_xyz[2]] = 1
+ret_val = affordance_map
+
+# Query: a point 5cm above the blue block.
+affordance_map = get_empty_affordance_map()
+blue_block = parse_query_obj('blue block')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = blue_block.aabb
+center_x, center_y, center_z = blue_block.position
+# 5cm above so we add to z-axis
+x = center_x
+y = center_y
+z = max_z + cm2index(5, 'z')
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 20cm away from the leftmost block.
+affordance_map = get_empty_affordance_map()
+leftmost_block = parse_query_obj('leftmost block')
+# positive normal because we are moving away from the block.
+moving_dir = leftmost_block.normal
+affordance_xyz = leftmost_block.position + cm2index(20, moving_dir)
+affordance_map[affordance_xyz[0], affordance_xyz[1], affordance_xyz[2]] = 1
+ret_val = affordance_map
+
+# Query: a point 4cm to the left of and 10cm on top of the tray that contains the lemon.
+affordance_map = get_empty_affordance_map()
+tray_with_lemon = parse_query_obj('tray that contains the lemon')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = tray_with_lemon.aabb
+center_x, center_y, center_z = tray_with_lemon.position
+# 4cm to the left of so we subtract from y-axis, and 10cm on top of so we add to z-axis
+x = center_x
+y = min_y - cm2index(4, 'y')
+z = max_z + cm2index(10, 'z')
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 10cm to the right of [45 49 66], and 5cm above it.
+affordance_map = get_empty_affordance_map()
+# 10cm to the right of so we add to y-axis, and 5cm above it so we add to z-axis
+x = 45
+y = 49 + cm2index(10, 'y')
+z = 66 + cm2index(5, 'z')
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: the blue circle.
+affordance_map = get_empty_affordance_map()
+blue_circle = parse_query_obj('blue circle')
+affordance_map = blue_circle.occupancy_map
+ret_val = affordance_map
+
+# Query: a point 10cm above and 5cm to the left of the yellow bowl.
+affordance_map = get_empty_affordance_map()
+yellow_bowl = parse_query_obj('yellow bowl')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = yellow_bowl.aabb
+center_x, center_y, center_z = yellow_bowl.position
+# 10cm above so we add to z-axis, and 5cm to the left of so we subtract from y-axis
+x = center_x
+y = min_y - cm2index(5, 'y')
+z = max_z + cm2index(10, 'z')
+affordance_map[x, y, z] = 1
+ret_val = affordance_map

VoxPoserApiExamples/Real/value_maps/real_get_avoidance_map_prompt.py

@@ -0,0 +1,36 @@
+import numpy as np
+from perception_utils import parse_query_obj
+from plan_utils import get_empty_avoidance_map, set_voxel_by_radius, cm2index
+
+# Query: 10cm from the bowl.
+avoidance_map = get_empty_avoidance_map()
+bowl = parse_query_obj('bowl')
+set_voxel_by_radius(avoidance_map, bowl.position, radius_cm=10, value=1)
+ret_val = avoidance_map
+
+# Query: 20cm near the mug.
+avoidance_map = get_empty_avoidance_map()
+mug = parse_query_obj('mug')
+set_voxel_by_radius(avoidance_map, mug.position, radius_cm=20, value=1)
+ret_val = avoidance_map
+
+# Query: 20cm around the mug and 10cm around the bowl.
+avoidance_map = get_empty_avoidance_map()
+mug = parse_query_obj('mug')
+set_voxel_by_radius(avoidance_map, mug.position, radius_cm=20, value=1)
+bowl = parse_query_obj('bowl')
+set_voxel_by_radius(avoidance_map, bowl.position, radius_cm=10, value=1)
+ret_val = avoidance_map
+
+# Query: 10cm from anything fragile.
+avoidance_map = get_empty_avoidance_map()
+fragile_objects = parse_query_obj('anything fragile')
+for obj in fragile_objects:
+    set_voxel_by_radius(avoidance_map, obj.position, radius_cm=10, value=1)
+ret_val = avoidance_map
+
+# Query: 10cm from the blue circle.
+avoidance_map = get_empty_avoidance_map()
+blue_circle = parse_query_obj('blue circle')
+set_voxel_by_radius(avoidance_map, blue_circle.position, radius_cm=10, value=1)
+ret_val = avoidance_map

VoxPoserApiExamples/Real/value_maps/real_get_gripper_map_prompt.py

@@ -0,0 +1,48 @@
+import numpy as np
+from perception_utils import parse_query_obj
+from plan_utils import get_empty_gripper_map, set_voxel_by_radius, cm2index
+
+# Query: open everywhere except 1cm around the green block.
+gripper_map = get_empty_gripper_map()
+# open everywhere
+gripper_map[:, :, :] = 0
+# close when 1cm around the green block
+green_block = parse_query_obj('green block')
+set_voxel_by_radius(gripper_map, green_block.position, radius_cm=1, value=1)
+ret_val = gripper_map
+
+# Query: close everywhere but open when on top of the back left corner of the table.
+gripper_map = get_empty_gripper_map()
+# close everywhere
+gripper_map[:, :, :] = 1
+# open when on top of the back left corner of the table
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# back so x = min_x, left so y = min_y, top so we add to z
+x = min_x
+y = min_y
+z = max_z + cm2index(15, 'z')
+set_voxel_by_radius(gripper_map, (x, y, z), radius_cm=10, value=0)
+ret_val = gripper_map
+
+# Query: always open except when you are on the right side of the table.
+gripper_map = get_empty_gripper_map()
+# always open
+gripper_map[:, :, :] = 0
+# close when you are on the right side of the table
+table = parse_query_obj('table')
+center_x, center_y, center_z = table.position
+# right side so y is greater than center_y
+gripper_map[:, center_y:, :] = 1
+
+# Query: always close except when you are on the back side of the table.
+gripper_map = get_empty_gripper_map()
+# always close
+gripper_map[:, :, :] = 1
+# open when you are on the back side of the table
+table = parse_query_obj('table')
+center_x, center_y, center_z = table.position
+# back side so x is less than center_x
+gripper_map[:center_x, :, :] = 0
+ret_val = gripper_map

VoxPoserApiExamples/Real/value_maps/real_get_rotation_map_prompt.py

@@ -0,0 +1,53 @@
+import numpy as np
+from plan_utils import get_empty_rotation_map, set_voxel_by_radius, cm2index, vec2quat
+from perception_utils import parse_query_obj
+from transforms3d.euler import euler2quat, quat2euler
+from transforms3d.quaternions import qmult, qinverse
+
+# Query: face the support surface of the bowl.
+rotation_map = get_empty_rotation_map()
+bowl = parse_query_obj('bowl')
+target_rotation = vec2quat(-bowl.normal)
+rotation_map[:, :, :] = target_rotation
+ret_val = rotation_map
+
+# Query: face the table when within 30cm from table center.
+rotation_map = get_empty_rotation_map()
+table = parse_query_obj('table')
+table_center = table.position
+target_rotation = vec2quat(-table.normal)
+set_voxel_by_radius(rotation_map, table_center, radius_cm=30, value=target_rotation)
+ret_val = rotation_map
+
+# Query: face the blue bowl.
+rotation_map = get_empty_rotation_map()
+blue_bowl = parse_query_obj('brown block')
+target_rotation = vec2quat(-blue_bowl.normal)
+rotation_map[:, :, :] = target_rotation
+ret_val = rotation_map
+
+# Query: turn clockwise by 45 degrees when at the center of the beer cap.
+rotation_map = get_empty_rotation_map()
+beer_cap = parse_query_obj('beer cap')
+(x, y, z) = beer_cap.position
+curr_rotation = rotation_map[x, y, z]
+rotation_delta = euler2quat(0, 0, np.pi / 4)
+rotation_map[x, y, z] = qmult(curr_rotation, rotation_delta)
+ret_val = rotation_map
+
+# Query: turn counter-clockwise by 30 degrees.
+rotation_map = get_empty_rotation_map()
+curr_rotation = rotation_map[0, 0, 0]
+rotation_delta = euler2quat(0, 0, -np.pi / 6)
+rotation_map[:, :, :] = qmult(curr_rotation, rotation_delta)
+ret_val = rotation_map
+
+# Query: rotate the gripper to be 45 degrees slanted relative to the plate.
+rotation_map = get_empty_rotation_map()
+plate = parse_query_obj('plate')
+face_plate_quat = vec2quat(-plate.normal)
+# rotate 45 degrees around the x-axis
+rotation_delta = euler2quat(-np.pi / 4, 0, 0)
+target_rotation = qmult(face_plate_quat, rotation_delta)
+rotation_map[:, :, :] = target_rotation
+ret_val = rotation_map

VoxPoserApiExamples/Real/value_maps/real_get_velocity_map_prompt.py

@@ -0,0 +1,31 @@
+import numpy as np
+from plan_utils import get_empty_velocity_map, set_voxel_by_radius, cm2index
+from perception_utils import parse_query_obj
+
+# Query: faster when on the right side of the table and slower when on the left side of the table.
+velocity_map = get_empty_velocity_map()
+table = parse_query_obj('table')
+center_x, center_y, center_z = table.position
+# faster on right side so 1.5 when y > center_y, slower on left side so 0.5 when y < center_y
+velocity_map[:, center_y:, :] = 1.5
+velocity_map[:, :center_y, :] = 0.5
+ret_val = velocity_map
+
+# Query: slow down by a quarter.
+velocity_map = get_empty_velocity_map()
+velocity_map[:] = 0.75
+ret_val = velocity_map
+
+# Query: slow down by a half when you're near anything fragile (objects: ['block', 'fork', 'mug', 'bowl', 'chips']).
+velocity_map = get_empty_velocity_map()
+mug = parse_query_obj('mug')
+set_voxel_by_radius(velocity_map, mug.position, radius_cm=10, value=0.5)
+bowl = parse_query_obj('bowl')
+set_voxel_by_radius(velocity_map, bowl.position, radius_cm=10, value=0.5)
+ret_val = velocity_map
+
+# Query: quarter of the speed when within 9cm from the yellow line.
+velocity_map = get_empty_velocity_map()
+yellow_line = parse_query_obj('yellow_line')
+set_voxel_by_radius(velocity_map, yellow_line.position, radius_cm=9, value=0.25)
+ret_val = velocity_map

VoxPoserApiExamples/Sim/composer/sim_composer_prompt.py

@@ -0,0 +1,91 @@
+import numpy as np
+from env_utils import execute, reset_to_default_pose
+from perception_utils import parse_query_obj
+from plan_utils import get_affordance_map, get_avoidance_map, get_velocity_map, get_rotation_map, get_gripper_map
+
+# Query: move ee forward for 7cm.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map(f'a point 7cm in front of {movable.position}')
+execute(movable, affordance_map)
+
+# Query: move to the left of the green block.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point 11cm to the left of the green block')
+execute(movable, affordance_map=affordance_map)
+
+# Query: move to the yellow block while staying on the front side the pink block.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point 11cm on top of the yellow block')
+avoidance_map = get_avoidance_map('anywhere behind the pink block')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map)
+
+# Query: move to the back side of the table.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point on the back side of the table')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map)
+
+# Query: move to the front right corner of the table while moving at faster speed when within 7cm from the yellow block.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point on the front right corner of the table')
+velocity_map = get_velocity_map('faster speed when within 7cm from the yellow block')
+execute(movable, affordance_map=affordance_map, velocity_map=velocity_map)
+
+# Query: move to 15cm on top of the rightmost block while avoiding the blue line and moving at a quarter of the speed on the right side of the table.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point 15cm on top of the rightmost block')
+avoidance_map = get_avoidance_map('the blue line')
+velocity_map = get_velocity_map('a quarter of the speed on the right side of the table')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map, velocity_map=velocity_map)
+
+# Query: close the topmost drawer by pushing.
+movable = parse_query_obj('topmost drawer handle')
+affordance_map = get_affordance_map('a point 21cm into the topmost drawer handle')
+execute(movable, affordance_map=affordance_map)
+
+# Query: push the second to the left block along the brown line.
+movable = parse_query_obj('second to the left block')
+affordance_map = get_affordance_map('the brown line')
+execute(movable, affordance_map=affordance_map)
+
+# Query: grasp the blue block from the table at a quarter of the speed.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point at the center of blue block')
+rotation_map = get_rotation_map('face the blue block')
+velocity_map = get_velocity_map('quarter of the speed')
+gripper_map = get_gripper_map('open everywhere except 1cm around the blue block')
+execute(movable, affordance_map=affordance_map, rotation_map=rotation_map, velocity_map=velocity_map, gripper_map=gripper_map)
+
+# Query: move to the left of the brown block.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point 11cm to the left of the brown block')
+execute(movable, affordance_map=affordance_map)
+
+# Query: back to default pose.
+reset_to_default_pose()
+
+# Query: open gripper.
+movable = parse_query_obj('gripper')
+gripper_map = get_gripper_map('open everywhere')
+execute(movable, gripper_map=gripper_map)
+
+# Query: drop the blue block to the right side of the table while staying at least 7cm away from the brown block.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point on the right side of the table')
+avoidance_map = get_avoidance_map('7cm away from the brown block')
+gripper_map = get_gripper_map('close everywhere except 1cm around the right side of the table')
+execute(movable, affordance_map=affordance_map, avoidance_map=avoidance_map, gripper_map=gripper_map)
+
+# Query: move to the front side of the yellow block.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point 11cm in front of the yellow block')
+execute(movable, affordance_map=affordance_map)
+
+# Query: move to the back side of the table.
+movable = parse_query_obj('ee')
+affordance_map = get_affordance_map('a point on the back side of the table')
+execute(movable, affordance_map=affordance_map)
+
+# Query: stay away from the brown block.
+movable = parse_query_obj('ee')
+avoidance_map = get_avoidance_map('3cm away from the brown block')
+execute(movable, avoidance_map=avoidance_map)

VoxPoserApiExamples/Sim/perception/sim_parse_query_obj_prompt.py

@@ -0,0 +1,71 @@
+import numpy as np
+from perception_utils import detect
+
+objects = ['green block', 'yellow line']
+# Query: ee.
+ee = detect('ee')[0]
+ret_val = ee
+
+objects = ['drawer', 'blue block', 'yellow block']
+# Query: topmost handle.
+handles = detect('drawer handle')
+# topmost so sort by z, take the last one
+handles = sorted(handles, key=lambda x: x.position[2])
+top_handle = handles[-1]
+ret_val = top_handle
+
+objects = ['yellow block', 'pink block', 'brown line', 'pink block']
+# Query: second to the left block.
+blocks = detect('block')
+# second to the left so sort by y, take the second one
+blocks = sorted(blocks, key=lambda x: x.position[1])
+second_left_block = blocks[1]
+ret_val = second_left_block
+
+objects = ['blue line', 'pink line', 'green block']
+# Query: table.
+table = detect('table')[0]
+ret_val = table
+
+objects = ['green line', 'drawer', 'yellow block']
+# Query: second to the bottom handle.
+handles = detect('drawer handle')
+# second to the bottom so sort by z, take the second one
+handles = sorted(handles, key=lambda x: x.position[2])
+second_bottom_handle = handles[1]
+ret_val = second_bottom_handle
+
+objects = ['brown line', 'brown block']
+# Query: brown line.
+brown_line = detect('brown line')[0]
+ret_val = brown_line
+
+objects = ['green block', 'brown block', 'yellow line']
+# Query: block.
+block = detect('green block')[0]
+ret_val = block
+
+objects = ['pink block', 'pink line', 'blue block']
+# Query: block closest to the pink line.
+blocks = detect('block')
+pink_line = detect('pink line')[0]
+closest_block = min(blocks, key=lambda x: np.linalg.norm(x.position - pink_line.position))
+ret_val = closest_block
+
+objects = ['blue block', 'blue line', 'green block', 'pink block', 'brown block']
+# Query: the block that is on top of the blue block.
+blocks = detect('block')
+blue_block = detect('blue block')[0]
+# find the block that is on top of the blue block
+for block in blocks:
+    if block.position[2] > blue_block.position[2]:
+        ret_val = block
+        break
+
+objects = ['drawer']
+# Query: top drawer handle.
+handles = detect('drawer handle')
+# top drawer handle so sort by z, take the last one
+handles = sorted(handles, key=lambda x: x.position[2])
+top_handle = handles[-1]
+ret_val = top_handle

VoxPoserApiExamples/Sim/value_maps/sim_get_affordance_map_prompt.py

@@ -0,0 +1,177 @@
+import numpy as np
+from perception_utils import parse_query_obj
+from plan_utils import get_empty_affordance_map, set_voxel_by_radius, cm2index
+
+# Query: a point 10cm in front of [10, 15, 60].
+affordance_map = get_empty_affordance_map()
+# in front so we add to x
+x = 10 + cm2index(10, 'x')
+y = 15
+z = 60
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point on the back side of the table.
+affordance_map = get_empty_affordance_map()
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# back side so x = min_x
+x = min_x
+y = center_y
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 21cm on top of the green block.
+affordance_map = get_empty_affordance_map()
+green_block = parse_query_obj('green block')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = green_block.aabb
+center_x, center_y, center_z = green_block.position
+# 21cm on top so z = max_z + 21cm
+x = center_x
+y = center_y
+z = max_z + cm2index(21, 'z')
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 1cm to the left of the brown block.
+affordance_map = get_empty_affordance_map()
+brown_block = parse_query_obj('brown block')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = brown_block.aabb
+center_x, center_y, center_z = brown_block.position
+# 1cm to the left so y = min_y - 1cm
+x = center_x
+y = min_y - cm2index(1, 'y')
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: anywhere within 15cm of the right most block.
+affordance_map = get_empty_affordance_map()
+right_most_block = parse_query_obj('the right most block')
+set_voxel_by_radius(affordance_map, right_most_block.position, radius_cm=15, value=1)
+ret_val = affordance_map
+
+# Query: a point on the back side of the table.
+affordance_map = get_empty_affordance_map()
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# back side so x = min_x
+x = min_x
+y = center_y
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point on the front right corner of the table.
+affordance_map = get_empty_affordance_map()
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# front right corner -> front so x = max_x, right so y = max_y
+x = max_x
+y = max_y
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 30cm into the topmost drawer handle.
+affordance_map = get_empty_affordance_map()
+top_handle = parse_query_obj('topmost drawer handle')
+# negative normal because we are moving into the handle.
+moving_dir = -top_handle.normal
+affordance_xyz = top_handle.position + cm2index(30, moving_dir)
+affordance_map[affordance_xyz[0], affordance_xyz[1], affordance_xyz[2]] = 1
+ret_val = affordance_map
+
+# Query: a point 9cm to the left of the blue block.
+affordance_map = get_empty_affordance_map()
+blue_block = parse_query_obj('blue block')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = blue_block.aabb
+center_x, center_y, center_z = blue_block.position
+# 9cm to the left so y = min_y - 9cm
+x = center_x
+y = min_y - cm2index(9, 'y')
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 21cm away from the leftmost block.
+affordance_map = get_empty_affordance_map()
+leftmost_block = parse_query_obj('leftmost block')
+# positive normal because we are moving away from the block.
+moving_dir = leftmost_block.normal
+affordance_xyz = leftmost_block.position + cm2index(21, moving_dir)
+affordance_map[affordance_xyz[0], affordance_xyz[1], affordance_xyz[2]] = 1
+ret_val = affordance_map
+
+# Query: a point 7cm above the tray that contains the lemon.
+affordance_map = get_empty_affordance_map()
+tray_with_lemon = parse_query_obj('tray that contains the lemon')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = tray_with_lemon.aabb
+center_x, center_y, center_z = tray_with_lemon.position
+# 7cm above so z = max_z + 7cm
+x = center_x
+y = center_y
+z = max_z + cm2index(7, 'z')
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 3cm to the left of [45, 49, 99].
+affordance_map = get_empty_affordance_map()
+# left so y = 49 - 3cm
+x = 45
+y = 49 - cm2index(3, 'y')
+z = 99
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: the brown line.
+brown_line = parse_query_obj('brown line')
+affordance_map = brown_line.occupancy_map
+ret_val = affordance_map
+
+# Query: a point 13cm to the left of the gripper.
+affordance_map = get_empty_affordance_map()
+gripper = parse_query_obj('gripper')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = gripper.aabb
+center_x, center_y, center_z = gripper.position
+# 13cm to the left so y = min_y - 13cm
+x = center_x
+y = min_y - cm2index(13, 'y')
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point 17cm above the back left corner of the table.
+affordance_map = get_empty_affordance_map()
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# 17cm on top of back left corner so x = min_x, y = min_y, z = max_z + 17cm
+x = min_x
+y = min_y
+z = max_z + cm2index(17, 'z')
+affordance_map[x, y, z] = 1
+ret_val = affordance_map
+
+# Query: a point at the center of the green block.
+affordance_map = get_empty_affordance_map()
+green_block = parse_query_obj('green block')
+center_x, center_y, center_z = green_block.position
+affordance_map[center_x, center_y, center_z] = 1
+ret_val = affordance_map
+
+# Query: the right side of the table.
+affordance_map = get_empty_affordance_map()
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# right side so y = max_y
+x = center_x
+y = max_y
+z = center_z
+affordance_map[x, y, z] = 1
+ret_val = affordance_map

VoxPoserApiExamples/Sim/value_maps/sim_get_avoidance_map_prompt.py

@@ -0,0 +1,60 @@
+import numpy as np
+from perception_utils import parse_query_obj
+from plan_utils import get_empty_avoidance_map, set_voxel_by_radius, cm2index
+
+# Query: 11cm from the blue block.
+avoidance_map = get_empty_avoidance_map()
+blue_block = parse_query_obj('blue block')
+set_voxel_by_radius(avoidance_map, blue_block.position, radius_cm=11, value=1)
+ret_val = avoidance_map
+
+# Query: 7cm near the pink block.
+avoidance_map = get_empty_avoidance_map()
+pink_block = parse_query_obj('pink block')
+set_voxel_by_radius(avoidance_map, pink_block.position, radius_cm=7, value=1)
+ret_val = avoidance_map
+
+# Query: 13cm around the brown block and 5cm around the green block.
+avoidance_map = get_empty_avoidance_map()
+brown_block = parse_query_obj('brown block')
+set_voxel_by_radius(avoidance_map, brown_block.position, radius_cm=13, value=1)
+green_block = parse_query_obj('green block')
+set_voxel_by_radius(avoidance_map, green_block.position, radius_cm=5, value=1)
+ret_val = avoidance_map
+
+# Query: the blue line.
+blue_line = parse_query_obj('blue_line')
+avoidance_map = blue_line.occupancy_map
+ret_val = avoidance_map
+
+# Query: anywhere on the front side of the blue block.
+avoidance_map = get_empty_avoidance_map()
+blue_block = parse_query_obj('blue block')
+center_x, center_y, center_z = blue_block.position
+# front side so x > center_x
+avoidance_map[center_x:, :, :] = 1
+ret_val = avoidance_map
+
+# Query: anywhere on the left of the green block.
+avoidance_map = get_empty_avoidance_map()
+green_block = parse_query_obj('green block')
+center_x, center_y, center_z = green_block.position
+# left side so y < center_y
+avoidance_map[:, :center_y, :] = 1
+ret_val = avoidance_map
+
+# Query: anywhere behind the pink block.
+avoidance_map = get_empty_avoidance_map()
+pink_block = parse_query_obj('pink block')
+center_x, center_y, center_z = pink_block.position
+# behind so x < center_x
+avoidance_map[:center_x, :, :] = 1
+ret_val = avoidance_map
+
+# Query: anywhere above the brown block.
+avoidance_map = get_empty_avoidance_map()
+brown_block = parse_query_obj('brown block')
+center_x, center_y, center_z = brown_block.position
+# above so z > center_z
+avoidance_map[:, :, center_z:] = 1
+ret_val = avoidance_map

VoxPoserApiExamples/Sim/value_maps/sim_get_gripper_map_prompt.py

@@ -0,0 +1,57 @@
+import numpy as np
+from perception_utils import parse_query_obj
+from plan_utils import get_empty_gripper_map, set_voxel_by_radius, cm2index
+
+# Query: open everywhere except 1cm around the green block.
+gripper_map = get_empty_gripper_map()
+# open everywhere
+gripper_map[:, :, :] = 0
+# close when 1cm around the green block
+green_block = parse_query_obj('green block')
+set_voxel_by_radius(gripper_map, green_block.position, radius_cm=1, value=1)
+ret_val = gripper_map
+
+# Query: close everywhere but open when on top of the back left corner of the table.
+gripper_map = get_empty_gripper_map()
+# close everywhere
+gripper_map[:, :, :] = 1
+# open when on top of the back left corner of the table
+table = parse_query_obj('table')
+(min_x, min_y, min_z), (max_x, max_y, max_z) = table.aabb
+center_x, center_y, center_z = table.position
+# back so x = min_x, left so y = min_y, top so we add to z
+x = min_x
+y = min_y
+z = max_z + cm2index(10, 'z')
+set_voxel_by_radius(gripper_map, (x, y, z), radius_cm=10, value=0)
+ret_val = gripper_map
+
+# Query: always open except when you are on the right side of the table.
+gripper_map = get_empty_gripper_map()
+# always open
+gripper_map[:, :, :] = 0
+# close when you are on the right side of the table
+table = parse_query_obj('table')
+center_x, center_y, center_z = table.position
+# right side so y is greater than center_y
+gripper_map[:, center_y:, :] = 1
+
+# Query: always close except when you are on the back side of the table.
+gripper_map = get_empty_gripper_map()
+# always close
+gripper_map[:, :, :] = 1
+# open when you are on the back side of the table
+table = parse_query_obj('table')
+center_x, center_y, center_z = table.position
+# back side so x is less than center_x
+gripper_map[:center_x, :, :] = 0
+ret_val = gripper_map
+
+# Query: open everywhere except 1cm around the topmost drawer handle.
+gripper_map = get_empty_gripper_map()
+# open everywhere
+gripper_map[:, :, :] = 0
+# close when 1cm around the topmost drawer handle
+topmost_drawer_handle = parse_query_obj('topmost drawer handle')
+set_voxel_by_radius(gripper_map, topmost_drawer_handle.position, radius_cm=1, value=1)
+ret_val = gripper_map

VoxPoserApiExamples/Sim/value_maps/sim_get_rotation_map_prompt.py

@@ -0,0 +1,25 @@
+import numpy as np
+from plan_utils import get_empty_rotation_map, set_voxel_by_radius, cm2index, vec2quat
+from perception_utils import parse_query_obj
+
+# Query: face the green block.
+rotation_map = get_empty_rotation_map()
+green_block = parse_query_obj('green block')
+target_rotation = vec2quat(-green_block.normal)
+rotation_map[:, :, :] = target_rotation
+ret_val = rotation_map
+
+# Query: face the table when within 30cm from table center.
+rotation_map = get_empty_rotation_map()
+table = parse_query_obj('table')
+table_center = table.position
+target_rotation = vec2quat(-table.normal)
+set_voxel_by_radius(rotation_map, table_center, radius_cm=30, value=target_rotation)
+ret_val = rotation_map
+
+# Query: face the topmost drawer handle.
+rotation_map = get_empty_rotation_map()
+topmost_drawer_handle = parse_query_obj('topmost drawer handle')
+target_rotation = vec2quat(-topmost_drawer_handle.normal)
+rotation_map[:, :, :] = target_rotation
+ret_val = rotation_map

VoxPoserApiExamples/Sim/value_maps/sim_get_velocity_map_prompt.py

@@ -0,0 +1,46 @@
+import numpy as np
+from plan_utils import get_empty_velocity_map, set_voxel_by_radius, cm2index
+from perception_utils import parse_query_obj
+
+# Query: faster when on the left side of the table and a quarter of the speed when on the other side of the table.
+velocity_map = get_empty_velocity_map()
+table = parse_query_obj('table')
+center_x, center_y, center_z = table.position
+# faster on left side so y < center_y
+velocity_map[:, :center_y, :] = 1.5
+# a quarter of the speed on the other side so y > center_y
+velocity_map[:, center_y:, :] = 0.25
+ret_val = velocity_map
+
+# Query: slow down by a quarter.
+velocity_map = get_empty_velocity_map()
+velocity_map[:] = 0.75
+ret_val = velocity_map
+
+# Query: quarter of the speed when within 9cm from the yellow block.
+velocity_map = get_empty_velocity_map()
+yellow_block = parse_query_obj('yellow block')
+set_voxel_by_radius(velocity_map, yellow_block.position, radius_cm=9, value=0.25)
+ret_val = velocity_map
+
+# Query: quarter of the speed in the back side of the table.
+velocity_map = get_empty_velocity_map()
+table = parse_query_obj('table')
+center_x, center_y, center_z = table.position
+# a quarter of the speed in the back side so x < center_x
+velocity_map[:center_x, :, :] = 0.25
+ret_val = velocity_map
+
+# Query: faster speed in the right side of the table.
+velocity_map = get_empty_velocity_map()
+table = parse_query_obj('table')
+center_x, center_y, center_z = table.position
+# faster in the right side so y > center_y
+velocity_map[:, center_y:, :] = 1.5
+ret_val = velocity_map
+
+# Query: quarter of the speed when within 11cm from the brown block.
+velocity_map = get_empty_velocity_map()
+brown_block = parse_query_obj('brown_block')
+set_voxel_by_radius(velocity_map, brown_block.position, radius_cm=11, value=0.25)
+ret_val = velocity_map

agibot_prompts/agibot_composer.py

@@ -0,0 +1,139 @@
+import numpy as np
+from env_utils import execute, reset_to_default_pose
+from perception_utils import search_object
+from plan_utils import get_target_pose, get_operation
+
+# Query: move ee forward for 10cm.
+while True:
+    movable = search_object('gripper')
+    target_pose = get_target_pose(f'a point 10cm in front of {movable.position}')
+    success = execute(movable, target_pose)
+    if success: break
+
+# Query: go back to default.
+reset_to_default_pose()
+
+# Query: move the gripper behind the bowl.
+while True:
+    movable = search_object('gripper')
+    target_pose = get_target_pose('a point 15cm behind the bowl')
+    success = execute(movable, target_pose=target_pose)
+    if success: break
+
+# Query: move to the back side of the table.
+while True:
+    movable = search_object('gripper')
+    target_pose = get_target_pose('a point on the back side of the table')
+    success = execute(movable, target_pose=target_pose)
+    if success: break
+
+# Query: move to the top of the plate.
+while True:
+    movable = search_object('gripper')
+    target_pose = get_target_pose('a point 10cm above the plate')
+    success = execute(movable, target_pose=target_pose)
+    if success: break
+
+# Query: drop the toy inside container.
+while True:
+    movable = search_object('gripper')
+    target_pose = get_target_pose('a point 15cm above the container')
+    operation = get_operation('close everywhere but open when on top of the container')
+    success = execute(movable, target_pose=target_pose, operation=operation)
+    if success: break
+
+# Query: push close the topmost drawer.
+while True:
+    movable = search_object('topmost drawer handle')
+    target_pose = get_target_pose('a point 30cm into the topmost drawer handle')
+    success = execute(movable, target_pose=target_pose)
+    if success: break
+
+# Query: push the second to the left block along the red line.
+while True:
+    movable = search_object('second to the left block')
+    target_pose = get_target_pose('the red line')
+    success = execute(movable, target_pose=target_pose)
+    if success: break
+
+# Query: grasp the blue block from the table.
+while True:
+    movable = search_object('gripper')
+    target_pose = get_target_pose('a point at the center of blue block')
+    operation = get_operation('open everywhere except 1cm around the blue block')
+    success = execute(movable, target_pose=target_pose, operation=operation)
+    if success: break
+
+# Query: move to the left of the brown block.
+while True:
+    movable = search_object('gripper')
+    target_pose = get_target_pose('a point 10cm to the left of the brown block')
+    success = execute(movable, target_pose=target_pose)
+    if success: break
+
+# Query: move to the top of the tray that contains the lemon.
+while True:
+    movable = search_object('gripper')
+    target_pose = get_target_pose('a point 10cm above the tray that contains the lemon')
+    success = execute(movable, target_pose=target_pose)
+    if success: break
+
+# Query: close drawer by 5cm.
+while True:
+    movable = search_object('drawer handle')
+    target_pose = get_target_pose('a point 5cm into the drawer handle')
+    success = execute(movable, target_pose=target_pose)
+    if success: break
+
+# Query: move to 5cm on top of the soda can, when within 20cm of the wooden mug.
+while True:
+    movable = search_object('gripper')
+    target_pose = get_target_pose('a point 5cm above the soda can')
+    success = execute(movable, target_pose=target_pose)
+    if success: break
+
+# Query: wipe the red dot.
+while True:
+    movable = search_object('gripper')
+    target_pose = get_target_pose('the red dot')
+    success = execute(movable, target_pose=target_pose)
+    if success: break
+
+# Query: grasp the mug from the shelf.
+while True:
+    movable = search_object('gripper')
+    target_pose = get_target_pose('a point at the center of the mug handle')
+    operation = get_operation('open everywhere except 1cm around the mug handle')
+    success = execute(movable, target_pose=target_pose, operation=operation)
+    if success: break
+
+# Query: move to 10cm on top of the soup bowl, and 5cm to the left of the soup bowl.
+while True:
+    movable = search_object('gripper')
+    target_pose = get_target_pose('a point 10cm above and 5cm to the left of the soup bowl')
+    success = execute(movable, target_pose=target_pose)
+    if success: break
+
+# Query: open gripper.
+while True:
+    movable = search_object('gripper')
+    operation = get_operation('open everywhere')
+    success = execute(movable, operation=operation)
+    if success: break
+
+# Query: sweep all particles to the left side of the table.
+particles = search_object('particles')
+for particle in particles:
+    while True:
+        movable = particle
+        target_pose = get_target_pose('a point on the left side of the table')
+        success = execute(particle, target_pose=target_pose)
+        if success: break
+
+# Query: grasp the bottom drawer handle.
+while True:
+    movable = search_object('gripper')
+    target_pose = get_target_pose('a point at the center of the bottom drawer handle')
+    operation = get_operation('open everywhere except 1cm around the bottom drawer handle')
+    success = execute(movable, target_pose=target_pose, operation=operation)
+    if success: break

agibot_prompts/agibot_perceptor.py

@@ -0,0 +1,104 @@
+import numpy as np
+from perception_utils import track
+
+objects = ['table', 'gripper', 'green block', 'cardboard box']
+# Query: gripper.
+gripper = track('gripper')[0]
+ret_val = gripper
+
+objects = ['table', 'gripper', 'drawer', 'egg', 'egg', 'plate']
+# Query: topmost handle.
+handles = track('drawer handle')
+handles = sorted(handles, key=lambda x: x.position[2])
+top_handle = handles[-1]
+ret_val = top_handle
+
+objects = ['table', 'gripper', 'yellow block', 'charging cable', 'cyan block', 'magenta block']
+# Query: second to the left block.
+blocks = track('block')
+blocks = sorted(blocks, key=lambda x: x.position[1])
+second_left_block = blocks[1]
+ret_val = second_left_block
+
+objects = ['table', 'gripper', 'iPhone', 'ruler', 'pink line', 'blue line']
+# Query: the front most line on the table.
+lines = track('line')
+lines = sorted(lines, key=lambda x: x.position[0])
+front_most_line = lines[-1]
+ret_val = front_most_line
+
+objects = ['table', 'gripper', 'vase', 'napkin box', 'mask']
+# Query: table.
+table = track('table')[0]
+ret_val = table
+
+objects = ['table', 'gripper', 'bottle', 'drawer', 'bowl', 'bag']
+# Query: second to the bottom handle.
+handles = track('drawer handle')
+handles = sorted(handles, key=lambda x: x.position[2])
+second_bottom_handle = handles[1]
+ret_val = second_bottom_handle
+
+objects = ['table', 'gripper', 'brown line', 'red block', 'monitor']
+# Query: brown line.
+brown_line = track('brown line')[0]
+ret_val = brown_line
+
+objects = ['table', 'gripper', 'green block', 'cup holder', 'black block']
+# Query: block.
+block = track('green block')[0]
+ret_val = block
+
+objects = ['table', 'gripper', 'mouse', 'yellow bowl', 'brown bowl', 'sticker']
+# Query: bowl closest to the sticker.
+bowls = track('bowl')
+sticker = track('sticker')[0]
+closest_bowl = min(bowls, key=lambda x: np.linalg.norm(x.position - sticker.position))
+ret_val = closest_bowl
+
+objects = ['table', 'gripper', 'keyboard', 'brown bag', 'pink bag', 'red tape', 'bottle']
+# Query: bag with the red tape on top.
+bags = track('bag')
+red_tape = track('red tape')[0]
+bag_with_red_tape = min(bags, key=lambda x: np.linalg.norm(x.position - red_tape.position))
+ret_val = bag_with_red_tape
+
+objects = ['table', 'gripper', 'grape', 'wood tray', 'strawberry', 'white tray', 'blue tray', 'bread']
+# Query: tray that contains the bread.
+trays = track('tray')
+bread = track('bread')[0]
+tray_with_bread = min(trays, key=lambda x: np.linalg.norm(x.position - bread.position))
+ret_val = tray_with_bread
+
+objects = ['table', 'gripper', 'drawer']
+# Query: top drawer handle.
+handles = track('drawer handle')
+top_drawer_handle = max(handles, key=lambda x: x.position[2])
+ret_val = top_drawer_handle
+
+objects = ['table', 'gripper', 'door']
+# Query: the thing you can open the door with.
+door_handle = track('door handle')[0]
+ret_val = door_handle
+
+objects = ['table', 'gripper', 'glass', 'vase', 'plastic bottle', 'block', 'phone case']
+# Query: anything fragile.
+fragile_items = []
+for obj in ['glass', 'vase']:
+    item = track(obj)[0]
+    fragile_items.append(item)
+ret_val = fragile_items
+
+objects = ['table', 'gripper', 'fridge']
+# Query: fridge handle.
+fridge_handle = track('fridge handle')[0]
+ret_val = fridge_handle
+
+objects = ['table', 'gripper', 'blue block', 'red block']
+# Query: green block.
+ret_val = None
+
+objects = ['table', 'gripper', 'yellow bowl', 'red spoon']
+# Query: gripper.
+gripper = track('gripper')[0]
+ret_val = gripper

agibot_prompts/agibot_planner.py

@@ -0,0 +1,121 @@
+import numpy as np
+from env_utils import execute
+from perception_utils import parse_query_obj
+import action_utils import composer
+
+objects = ['blue block', 'yellow block', 'mug']
+# Query: place the blue block on the yellow block, and avoid the mug at all time.
+composer("grasp the blue block while keeping at least 15cm away from the mug")
+composer("back to default pose")
+composer("move to 5cm on top of the yellow block while keeping at least 15cm away from the mug")
+composer("open gripper")
+# done
+
+objects = ['airpods', 'drawer']
+# Query: Open the drawer slowly.
+composer("grasp the drawer handle, at 0.5x speed")
+composer("move away from the drawer handle by 25cm, at 0.5x speed")
+composer("open gripper, at 0.5x speed")
+# done
+
+objects = ['tissue box', 'tissue', 'bowl']
+# Query: Can you pass me a tissue and place it next to the bowl?
+composer("grasp the tissue")
+composer("back to default pose")
+composer("move to 10cm to the right of the bowl")
+composer("open gripper")
+composer("back to default pose")
+# done
+
+objects = ['charger', 'outlet']
+# Query: unplug the charger from the wall.
+composer("grasp the charger")
+composer("back to default pose")
+# done
+
+objects = ['grape', 'lemon', 'drill', 'router', 'bread', 'tray']
+# Query: put the sweeter fruit in the tray that contains the bread.
+composer("grasp the grape")
+composer("back to default pose")
+composer("move to the top of the tray that contains the bread")
+composer("open gripper")
+# done
+
+objects = ['marbles', 'tray', 'broom']
+# Query: Can you sweep the marbles into the tray?
+composer("grasp the broom")
+composer("back to default pose")
+composer("push the marbles into the tray")
+# done
+
+objects = ['orange', 'QR code', 'lemon', 'drawer']
+# Query: put the sour fruit into the top drawer.
+composer("grasp the top drawer handle")
+composer("move away from the top drawer handle by 25cm")
+composer("open gripper")
+composer("back to default pose")
+composer("grasp the lemon")
+composer("move to 10cm on top of the top drawer")
+composer("open gripper")
+# done
+
+objects = ['fridge', 'hot soup']
+# Query: Open the fridge door and be careful around the hot soup.
+composer("grasp the fridge handle and keep at least 15cm away from the hot soup")
+composer("move away from the fridge handle by 25cm and keep at least 15cm away from the hot soup")
+composer("open gripper")
+# done
+
+objects = ['cyan bowl', 'yellow bowl', 'box', 'ice cream']
+# Query: move to the top of the cyan bowl.
+composer("move to the top of the cyan bowl")
+# done
+
+objects = ['drawer', 'umbrella']
+# Query: close the drawer.
+composer("push close the drawer handle by 25cm")
+# done
+
+objects = ['plate', 'steak', 'fork', 'knife', 'spoon']
+# Query: Could you please set up the fork for the steak for me?
+composer("grasp the fork")
+composer("back to default pose")
+composer("move to 10cm to the right of the plate")
+composer("open gripper")
+composer("back to default pose")
+# done
+
+objects = ['light switch']
+# Query: Press down the switch.
+composer("close the gripper")
+composer("move to the center of the light switch")
+composer("back to default pose")
+# done
+
+objects = ['beer']
+# Query: turn close the beer.
+composer("grasp the beer cap")
+composer("turn clockwise by 180 degrees")
+composer("back to default pose")
+# done
+
+objects = ['steak', 'grill', 'plate']
+# Query: Take the steak out of the grill and put it flat on the plate.
+composer("grasp the steak")
+composer("back to default pose")
+composer("rotate the gripper to be 45 degrees slanted relative to the plate")
+composer("move to 10cm on top of the plate")
+composer("open gripper")
+composer("back to default pose")
+# done
+
+objects = ['trash can', 'napkin']
+# Query: sort napkin into trash can.
+napkin = parse_query_obj("napkin")
+trash_can = parse_query_obj("trash can")
+while np.linalg.norm(napkin.position - trash_can.position) > 0.1:
+    composer("grasp the napkin")
+    composer("move to 10cm on top of the trash can")
+    composer("open gripper")
+    composer("back to default pose")
+# done

app.py

@@ -0,0 +1,202 @@
+import openai
+import numpy as np
+from tempfile import NamedTemporaryFile
+import copy
+import shapely
+from shapely.geometry import *
+from shapely.affinity import *
+from omegaconf import OmegaConf
+from moviepy.editor import ImageSequenceClip
+import gradio as gr
+
+from lmp import LMP, LMPFGen
+from sim import PickPlaceEnv, LMP_wrapper, VoxPoserWrapper, AgibotWrapper
+from consts import ALL_BLOCKS, ALL_BOWLS
+from md_logger import MarkdownLogger
+from utils import get_geoip
+
+class DemoRunner:
+
+    def __init__(self, config_file: str = 'cfg.yaml'):
+        self._cfg = OmegaConf.to_container(OmegaConf.load(config_file), resolve=True)
+        self._env = None
+        self._md_logger = MarkdownLogger()
+
+    def make_LMP(self, env, cfg_choice):
+        # LMP env wrapper
+        cfg = copy.deepcopy(self._cfg)
+        # cfg['env'] = {
+        #     'init_objs': list(env.obj_name_to_id.keys()),
+        #     'coords': cfg['tabletop_coords']
+        # }
+
+        if cfg_choice == "voxposer":
+            LMP_env = VoxPoserWrapper(env, cfg)
+        elif cfg_choice == "agibot":
+            LMP_env = AgibotWrapper(env, cfg)
+
+        # creating APIs that the LMPs can interact with
+        fixed_vars = {
+            'np': np
+        }
+        fixed_vars.update({
+            name: eval(name)
+            for name in shapely.geometry.__all__ + shapely.affinity.__all__
+        })
+
+        variable_vars = {
+            k: getattr(LMP_env, k)
+            for k in dir(LMP_env)
+            if not k.startswith("__") and callable(getattr(LMP_env, k))
+        }
+        variable_vars['say'] = lambda msg: self._md_logger.log_text(f'Robot says: "{msg}"')
+
+        # creating the function-generating LMP
+        lmp_fgen = LMPFGen(cfg['lmps']['fgen'], fixed_vars, variable_vars, self._md_logger)
+
+        # creating other low-level LMPs
+        variable_vars.update({
+            k: LMP(k, cfg['lmps'][k], lmp_fgen, fixed_vars, variable_vars, self._md_logger)
+            for k in cfg['lmps'].keys() if k != 'fgen'
+        })
+
+        # creating the LMP that deals w/ high-level language commands
+        lmp_planner = LMP(
+            'planner', cfg['lmps']['planner'], lmp_fgen, fixed_vars, variable_vars, self._md_logger
+        )
+
+        return lmp_planner
+
+    def setup(self, api_key, n_blocks, n_bowls, proxy, cfg_choice):
+        openai.api_key = api_key
+
+        # self._env = PickPlaceEnv(render=True, high_res=True, high_frame_rate=False)
+        # list_idxs = np.random.choice(len(ALL_BLOCKS), size=max(n_blocks, n_bowls), replace=False)
+        # block_list = [ALL_BLOCKS[i] for i in list_idxs[:n_blocks]]
+        # bowl_list = [ALL_BOWLS[i] for i in list_idxs[:n_bowls]]
+        # obj_list = block_list + bowl_list
+        # self._env.reset(obj_list)
+
+        self._lmp_planner = self.make_LMP(self._env, cfg_choice)
+
+        # info = '### Available Objects: \n- ' + '\n- '.join(obj_list)
+        # img = self._env.get_camera_image()
+
+        info, img = None, None
+
+        return info, img
+
+    def run(self, instruction):
+        # if self._env is None:
+        #     return 'Please run setup first!', None, None
+
+        # self._env.cache_video = []
+        self._md_logger.clear()
+
+        self._lmp_planner(instruction)
+
+        # try:
+        #     self._lmp_planner(instruction, f'objects = {self._env.object_list}')
+        # except Exception as e:
+        #     return f'Error: {e}', None, None
+
+        # video_file_name = None
+        # if self._env.cache_video:
+        #     rendered_clip = ImageSequenceClip(self._env.cache_video, fps=25)
+        #     video_file_name = NamedTemporaryFile(suffix='.mp4').name
+        #     rendered_clip.write_videofile(video_file_name, fps=25)
+        video_file_name = None
+
+        # return self._md_logger.get_log(), self._env.get_camera_image(), video_file_name
+
+        return self._md_logger.get_log(), None, video_file_name
+
+
+def setup(api_key, n_blocks, n_bowls, proxy_addr, cfg_choice):
+    if not api_key:
+        return 'Please enter your OpenAI API key!', None, None
+
+    if not proxy_addr:
+        return 'Please enter your local proxy address!', None, None
+
+    if n_blocks + n_bowls == 0:
+        return 'Please select at least one object!', None, None
+
+    if "http://" in proxy_addr:
+        openai.proxy = proxy_addr
+    else:
+        openai.proxy = f"http://{proxy_addr}"
+
+    # ip_status, ip_info = get_geoip(openai.proxy)
+    # if ip_status == -1:
+    #     return ip_info, None, None
+    # elif ip_status == 0:
+    #     pressed_key = input('Continue with current ip location? (y/n)')
+    #     if pressed_key.lower() != 'y':
+    #         return ip_info, None, None
+    # else:
+    #     print(f'{ip_info} IP location check passed.')
+
+    if cfg_choice == "voxposer":
+        cfg_file = 'cfg_voxposer.yaml'
+    elif cfg_choice == "agibot":
+        cfg_file = 'cfg_agibot.yaml'
+
+    demo_runner = DemoRunner(cfg_file)
+
+    info, img = demo_runner.setup(api_key, n_blocks, n_bowls, proxy_addr, cfg_choice)
+    return info, img, demo_runner
+
+
+def run(instruction, demo_runner):
+    if demo_runner is None:
+        return 'Please run setup first!', None, None
+    return demo_runner.run(instruction)
+
+
+if __name__ == '__main__':
+    with open('README.md', 'r') as f:
+        for _ in range(12):
+            next(f)
+        readme_text = f.read()
+
+    with gr.Blocks() as demo:
+        state = gr.State(None)
+
+        # gr.Markdown(readme_text)
+        gr.Markdown('# Interactive Demo')
+        with gr.Row():
+            with gr.Column():
+                with gr.Column():
+                    inp_api_key = gr.Textbox(label='OpenAI API Key (this is not stored anywhere)', lines=1)
+                    inp_proxy_addr = gr.Textbox(label='Your local proxy address', lines=1)
+                    inp_cfg = gr.Dropdown(label='Configuration', choices=['voxposer', 'agibot'])
+                with gr.Row():
+                    inp_n_blocks = gr.Slider(label='Number of Blocks', minimum=0, maximum=4, value=3, step=1)
+                    inp_n_bowls = gr.Slider(label='Number of Bowls', minimum=0, maximum=4, value=3, step=1)
+
+                btn_setup = gr.Button("Setup/Reset Simulation")
+                info_setup = gr.Markdown(label='Setup Info')
+            with gr.Column():
+                img_setup = gr.Image(label='Current Simulation')
+
+        with gr.Row():
+            with gr.Column():
+                inp_instruction = gr.Textbox(label='Instruction', lines=1)
+                btn_run = gr.Button("Run (this may take 30+ seconds)")
+                info_run = gr.Markdown(label='Generated Code')
+            with gr.Column():
+                video_run = gr.Video(label='Video of Last Instruction')
+
+        btn_setup.click(
+            setup,
+            inputs=[inp_api_key, inp_n_blocks, inp_n_bowls, inp_proxy_addr, inp_cfg],
+            outputs=[info_setup, img_setup, state]
+        )
+        btn_run.click(
+            run,
+            inputs=[inp_instruction, state],
+            outputs=[info_run, img_setup, video_run]
+        )
+
+    demo.launch()

bowl/bowl.urdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:505058761a89c12542f9ba28c04061774a4016a65b53c3bfd822cf5a9dd19d1b
+size 806

bowl/cup.obj

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:624cba3eaf09307b766ef791eea49a188a28f8969ec8b28c724ae1df4d4ce446
+size 27464

bowl/textured-0008192.obj

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fee6d4c3f8b4fe40de17801e9a475463c435b9e3f151fc5a9bcfc5a91355f2a
+size 724293

cfg.yaml

@@ -0,0 +1,90 @@
+
+lmps:
+  tabletop_ui:
+    prompt_path: prompts/tabletop_ui.py
+    engine: text-davinci-003
+    max_tokens: 256
+    temperature: 0
+    query_prefix: '# '
+    query_suffix: '.'
+    stop: ['#', 'objects = [']
+    maintain_session: True
+    debug_mode: False
+    include_context: True
+    has_return: False
+    return_val_name: ret_val
+  parse_obj_name:
+    prompt_path: prompts/parse_obj_name.py
+    engine: text-davinci-003
+    max_tokens: 512
+    temperature: 0
+    query_prefix: '# '
+    query_suffix: '.'
+    stop: ['#', 'objects = [']
+    maintain_session: False
+    debug_mode: False
+    include_context: True
+    has_return: True
+    return_val_name: ret_val
+  parse_position:
+    prompt_path: prompts/parse_position.py
+    engine: text-davinci-003
+    max_tokens: 512
+    temperature: 0
+    query_prefix: '# '
+    query_suffix: '.'
+    stop: ['#']
+    maintain_session: False
+    debug_mode: False
+    include_context: True
+    has_return: True
+    return_val_name: ret_val
+  parse_question:
+    prompt_path: prompts/parse_question.py
+    engine: text-davinci-003
+    max_tokens: 512
+    temperature: 0
+    query_prefix: '# '
+    query_suffix: '.'
+    stop: ['#', 'objects = [']
+    maintain_session: False
+    debug_mode: False
+    include_context: True
+    has_return: True
+    return_val_name: ret_val
+  transform_shape_pts:
+    prompt_path: prompts/transform_shape_pts.py
+    engine: text-davinci-003
+    max_tokens: 512
+    temperature: 0
+    query_prefix: '# '
+    query_suffix: '.'
+    stop: ['#']
+    maintain_session: False
+    debug_mode: False
+    include_context: True
+    has_return: True
+    return_val_name: new_shape_pts
+  fgen:
+    prompt_path: prompts/fgen.py
+    engine: text-davinci-003
+    max_tokens: 512
+    temperature: 0
+    query_prefix: '# define function: '
+    query_suffix: '.'
+    stop: ['# define', '# example']
+    maintain_session: False
+    debug_mode: False
+    include_context: True
+
+tabletop_coords:
+  top_left: [-0.25, -0.25]
+  top_side: [0, -0.25]
+  top_right: [0.25, -0.25]
+  left_side: [-0.25, -0.5]
+  middle: [0, -0.5]
+  right_side: [0.25, -0.5]
+  bottom_left: [-0.25, -0.75]
+  bottom_side: [0, -0.75]
+  bottom_right: [0.25, -0.75]
+  table_z: 0

cfg_agibot.yaml

@@ -0,0 +1,51 @@
+lmps:
+  planner:
+    prompt_path: agibot_prompts/agibot_planner.py
+    model: gpt-4
+    max_tokens: 1024
+    temperature: 0
+    query_prefix: '# Query: '
+    query_suffix: '.'
+    stop: ['#', 'objects = [']
+    maintain_session: True
+    debug_mode: False
+    include_context: True
+    has_return: False
+    return_val_name: ret_val
+  composer:
+    prompt_path: agibot_prompts/agibot_composer.py
+    model: gpt-4
+    max_tokens: 1024
+    temperature: 0
+    query_prefix: '# Query: '
+    query_suffix: '.'
+    stop: ['#', 'objects = [']
+    maintain_session: False
+    debug_mode: False
+    include_context: True
+    has_return: False
+    return_val_name: ret_val
+  search_object:
+    prompt_path: agibot_prompts/agibot_perceptor.py
+    model: gpt-4
+    max_tokens: 512
+    temperature: 0
+    query_prefix: '# Query: '
+    query_suffix: '.'
+    stop: ['#']
+    maintain_session: False
+    debug_mode: True
+    include_context: True
+    has_return: True
+    return_val_name: ret_val
+  fgen:
+    prompt_path: prompts/fgen.py
+    engine: gpt-4
+    max_tokens: 512
+    temperature: 0
+    query_prefix: '# define function: '
+    query_suffix: '.'
+    stop: ['# define', '# example']
+    maintain_session: False
+    debug_mode: False
+    include_context: True

cfg_voxposer.yaml

@@ -0,0 +1,114 @@
+
+lmps:
+  planner:
+    prompt_path: VoxPoserApiExamples/Real/planner/real_planner_prompt.py
+    model: gpt-4
+    max_tokens: 1024
+    temperature: 0
+    query_prefix: '# Query: '
+    query_suffix: '.'
+    stop: ['#', 'objects = [']
+    maintain_session: True
+    debug_mode: False
+    include_context: True
+    has_return: False
+    return_val_name: ret_val
+  composer:
+    prompt_path: VoxPoserApiExamples/Real/composer/real_composer_prompt.py
+    model: gpt-4
+    max_tokens: 1024
+    temperature: 0
+    query_prefix: '# Query: '
+    query_suffix: '.'
+    stop: ['#', 'objects = [']
+    maintain_session: False
+    debug_mode: False
+    include_context: True
+    has_return: False
+    return_val_name: ret_val
+  get_affordance_map:
+    prompt_path: VoxPoserApiExamples/Real/value_maps/real_get_affordance_map_prompt.py
+    model: gpt-4
+    max_tokens: 512
+    temperature: 0
+    query_prefix: '# Query: '
+    query_suffix: '.'
+    stop: ['#']
+    maintain_session: False
+    debug_mode: True
+    include_context: True
+    has_return: True
+    return_val_name: ret_val
+  get_avoidance_map:
+    prompt_path: VoxPoserApiExamples/Real/value_maps/real_get_avoidance_map_prompt.py
+    model: gpt-4
+    max_tokens: 512
+    temperature: 0
+    query_prefix: '# Query: '
+    query_suffix: '.'
+    stop: ['#', 'objects = [']
+    maintain_session: False
+    debug_mode: True
+    include_context: True
+    has_return: True
+    return_val_name: ret_val
+  get_gripper_map:
+    prompt_path: VoxPoserApiExamples/Real/value_maps/real_get_gripper_map_prompt.py
+    model: gpt-4
+    max_tokens: 512
+    temperature: 0
+    query_prefix: '# Query: '
+    query_suffix: '.'
+    stop: ['#']
+    maintain_session: False
+    debug_mode: True
+    include_context: True
+    has_return: True
+    return_val_name: new_shape_pts
+  get_rotation_map:
+    prompt_path: VoxPoserApiExamples/Real/value_maps/real_get_rotation_map_prompt.py
+    model: gpt-4
+    max_tokens: 512
+    temperature: 0
+    query_prefix: '# define function: '
+    query_suffix: '.'
+    stop: ['# define', '# example']
+    maintain_session: False
+    debug_mode: True
+    include_context: True
+    has_return: True
+  get_velocity_map:
+    prompt_path: VoxPoserApiExamples/Real/value_maps/real_get_velocity_map_prompt.py
+    model: gpt-4
+    max_tokens: 512
+    temperature: 0
+    query_prefix: '# define function: '
+    query_suffix: '.'
+    stop: ['# define', '# example']
+    maintain_session: False
+    debug_mode: True
+    include_context: True
+    has_return: True
+  fgen:
+    prompt_path: prompts/fgen.py
+    engine: gpt-4
+    max_tokens: 512
+    temperature: 0
+    query_prefix: '# define function: '
+    query_suffix: '.'
+    stop: ['# define', '# example']
+    maintain_session: False
+    debug_mode: False
+    include_context: True
+
+tabletop_coords:
+  top_left: [-0.25, -0.25]
+  top_side: [0, -0.25]
+  top_right: [0.25, -0.25]
+  left_side: [-0.25, -0.5]
+  middle: [0, -0.5]
+  right_side: [0.25, -0.5]
+  bottom_left: [-0.25, -0.75]
+  bottom_side: [0, -0.75]
+  bottom_right: [0.25, -0.75]
+  table_z: 0

consts.py

@@ -0,0 +1,33 @@
+import numpy as np
+
+# # Global constants: pick and place objects, colors, workspace bounds
+COLORS = {
+    'blue':   (78/255,  121/255, 167/255, 255/255),
+    'red':    (255/255,  87/255,  89/255, 255/255),
+    'green':  (89/255,  169/255,  79/255, 255/255),
+    'orange': (242/255, 142/255,  43/255, 255/255),
+    'yellow': (237/255, 201/255,  72/255, 255/255),
+    'purple': (176/255, 122/255, 161/255, 255/255),
+    'pink':   (255/255, 157/255, 167/255, 255/255),
+    'cyan':   (118/255, 183/255, 178/255, 255/255),
+    'brown':  (156/255, 117/255,  95/255, 255/255),
+    'gray':   (186/255, 176/255, 172/255, 255/255),
+}
+
+CORNER_POS = {
+  'top left corner':     (-0.3 + 0.05, -0.2 - 0.05, 0),
+  'top side':            (0,           -0.2 - 0.05, 0),
+  'top right corner':    (0.3 - 0.05,  -0.2 - 0.05, 0),
+  'left side':           (-0.3 + 0.05, -0.5,        0),
+  'middle':              (0,           -0.5,        0),
+  'right side':          (0.3 - 0.05,  -0.5,        0),
+  'bottom left corner':  (-0.3 + 0.05, -0.8 + 0.05, 0),
+  'bottom side':         (0,           -0.8 + 0.05, 0),
+  'bottom right corner': (0.3 - 0.05,  -0.8 + 0.05, 0),
+}
+
+ALL_BLOCKS = ['blue block', 'red block', 'green block', 'orange block', 'yellow block', 'purple block', 'pink block', 'cyan block', 'brown block', 'gray block']
+ALL_BOWLS = ['blue bowl', 'red bowl', 'green bowl', 'orange bowl', 'yellow bowl', 'purple bowl', 'pink bowl', 'cyan bowl', 'brown bowl', 'gray bowl']
+
+PIXEL_SIZE = 0.00267857
+BOUNDS = np.float32([[-0.3, 0.3], [-0.8, -0.2], [0, 0.15]])  # X Y Z

lmp.py

@@ -0,0 +1,253 @@
+from time import sleep
+import ast
+import astunparse
+import openai
+from openai.error import RateLimitError, APIConnectionError
+from pygments import highlight
+from pygments.lexers import PythonLexer
+from pygments.formatters import TerminalFormatter
+
+from utils import retrieve_proxy, chat_completion_request, generate_messages
+
+class LMP:
+
+    def __init__(self, name, cfg, lmp_fgen, fixed_vars, variable_vars, md_logger):
+        self._name = name
+        self._cfg = cfg
+        self._md_logger = md_logger
+
+        with open(self._cfg['prompt_path'], 'r') as f:
+            self._base_prompt = f.read()
+
+        self._stop_tokens = list(self._cfg['stop'])
+
+        self._lmp_fgen = lmp_fgen
+
+        self._fixed_vars = fixed_vars
+        self._variable_vars = variable_vars
+        self.exec_hist = ''
+
+    def clear_exec_hist(self):
+        self.exec_hist = ''
+
+    def build_prompt(self, query, context=''):
+        if len(self._variable_vars) > 0:
+            variable_vars_imports_str = f"from utils import {', '.join(self._variable_vars.keys())}"
+        else:
+            variable_vars_imports_str = ''
+        prompt = self._base_prompt.replace('{variable_vars_imports}', variable_vars_imports_str)
+
+        if self._cfg['maintain_session']:
+            prompt += f'\n{self.exec_hist}'
+
+        if context != '':
+            prompt += f'\n{context}'
+
+        use_query = f'{self._cfg["query_prefix"]}{query}{self._cfg["query_suffix"]}'
+        prompt += f'\n{use_query}'
+
+        return prompt, use_query
+
+    def __call__(self, query, context='', **kwargs):
+        prompt, use_query = self.build_prompt(query, context=context)
+
+        response = chat_completion_request(
+            generate_messages(prompt),
+            # stop=self._stop_tokens,
+            temperature=self._cfg['temperature'],
+            model=self._cfg['model'],
+            max_tokens=self._cfg['max_tokens'],
+            proxy=openai.proxy,
+        )
+        code_str = response.json()["choices"][0]["message"]['content']
+
+        if self._cfg['include_context'] and context != '':
+            to_exec = f'{context}\n{code_str}'
+            to_log = f'{context}\n{use_query}\n{code_str}'
+        else:
+            to_exec = code_str
+            to_log = f'{use_query}\n{to_exec}'
+
+        to_log_pretty = highlight(to_log, PythonLexer(), TerminalFormatter())
+        print(f'LMP {self._name} generated code:\n{to_log_pretty}')
+        self._md_logger.log_text(f'LMP {self._name} Generated Code:')
+        self._md_logger.log_code(to_log)
+
+        new_fs = self._lmp_fgen.create_new_fs_from_code(code_str)
+        self._variable_vars.update(new_fs)
+
+        gvars = merge_dicts([self._fixed_vars, self._variable_vars])
+        lvars = kwargs
+
+        if not self._cfg['debug_mode']:
+            exec_safe(to_exec, gvars, lvars)
+
+        self.exec_hist += f'\n{to_exec}'
+
+        if self._cfg['maintain_session']:
+            self._variable_vars.update(lvars)
+
+        if self._cfg['has_return'] and not self._cfg['debug_mode']:
+            return lvars[self._cfg['return_val_name']]
+
+class LMPFGen:
+
+    def __init__(self, cfg, fixed_vars, variable_vars, md_logger):
+        self._cfg = cfg
+
+        self._stop_tokens = list(self._cfg['stop'])
+        self._fixed_vars = fixed_vars
+        self._variable_vars = variable_vars
+        self._md_logger = md_logger
+
+        with open(self._cfg['prompt_path'], 'r') as f:
+            self._base_prompt = f.read()
+
+    def create_f_from_sig(self, f_name, f_sig, other_vars=None, fix_bugs=False, return_src=False):
+        print(f'Creating function: {f_sig}')
+
+        use_query = f'{self._cfg["query_prefix"]}{f_sig}{self._cfg["query_suffix"]}'
+        prompt = f'{self._base_prompt}\n{use_query}'
+
+        response = chat_completion_request(
+            generate_messages(prompt),
+            temperature=self._cfg['temperature'],
+            model=self._cfg['model'],
+            max_tokens=self._cfg['max_tokens'],
+            proxy=openai.proxy,
+        )
+        f_src = response.json()["choices"][0]["message"]['content']
+
+        if fix_bugs:
+            f_src = openai.Edit.create(
+                model='code-davinci-edit-001',
+                input='# ' + f_src,
+                temperature=0,
+                instruction='Fix the bug if there is one. Improve readability. Keep same inputs and outputs. Only small changes. No comments.',
+            )['choices'][0]['text'].strip()
+
+        if other_vars is None:
+            other_vars = {}
+        gvars = merge_dicts([self._fixed_vars, self._variable_vars, other_vars])
+        lvars = {}
+        
+        exec_safe(f_src, gvars, lvars)
+
+        f = lvars[f_name]
+
+        to_print = f'{use_query}\n{f_src}'
+        to_print_pretty = highlight(to_print, PythonLexer(), TerminalFormatter())
+        print(f'LMPFGen generated code:\n{to_print_pretty}')
+        self._md_logger.log_text('Generated Function:')
+        self._md_logger.log_code(to_print)
+
+        if return_src:
+            return f, f_src
+        return f
+
+    def create_new_fs_from_code(self, code_str, other_vars=None, fix_bugs=False, return_src=False):
+        fs, f_assigns = {}, {}
+        f_parser = FunctionParser(fs, f_assigns)
+        f_parser.visit(ast.parse(code_str))
+        for f_name, f_assign in f_assigns.items():
+            if f_name in fs:
+                fs[f_name] = f_assign
+
+        if other_vars is None:
+            other_vars = {}
+
+        new_fs = {}
+        srcs = {}
+        for f_name, f_sig in fs.items():
+            all_vars = merge_dicts([self._fixed_vars, self._variable_vars, new_fs, other_vars])
+            if not var_exists(f_name, all_vars):
+                f, f_src = self.create_f_from_sig(f_name, f_sig, new_fs, fix_bugs=fix_bugs, return_src=True)
+
+                # recursively define child_fs in the function body if needed
+                f_def_body = astunparse.unparse(ast.parse(f_src).body[0].body)
+                child_fs, child_f_srcs = self.create_new_fs_from_code(
+                    f_def_body, other_vars=all_vars, fix_bugs=fix_bugs, return_src=True
+                )
+
+                if len(child_fs) > 0:
+                    new_fs.update(child_fs)
+                    srcs.update(child_f_srcs)
+
+                    # redefine parent f so newly created child_fs are in scope
+                    gvars = merge_dicts([self._fixed_vars, self._variable_vars, new_fs, other_vars])
+                    lvars = {}
+                    
+                    exec_safe(f_src, gvars, lvars)
+                    
+                    f = lvars[f_name]
+
+                new_fs[f_name], srcs[f_name] = f, f_src
+
+        if return_src:
+            return new_fs, srcs
+        return new_fs
+
+
+class FunctionParser(ast.NodeTransformer):
+
+    def __init__(self, fs, f_assigns):
+      super().__init__()
+      self._fs = fs
+      self._f_assigns = f_assigns
+
+    def visit_Call(self, node):
+        self.generic_visit(node)
+        if isinstance(node.func, ast.Name):
+            f_sig = astunparse.unparse(node).strip()
+            f_name = astunparse.unparse(node.func).strip()
+            self._fs[f_name] = f_sig
+        return node
+
+    def visit_Assign(self, node):
+        self.generic_visit(node)
+        if isinstance(node.value, ast.Call):
+            assign_str = astunparse.unparse(node).strip()
+            f_name = astunparse.unparse(node.value.func).strip()
+            self._f_assigns[f_name] = assign_str
+        return node
+
+
+def var_exists(name, all_vars):
+    try:
+        eval(name, all_vars)
+    except:
+        exists = False
+    else:
+        exists = True
+    return exists
+
+
+def merge_dicts(dicts):
+    return {
+        k : v 
+        for d in dicts
+        for k, v in d.items()
+    }
+
+
+import traceback
+def exec_safe(code_str, gvars=None, lvars=None):
+    banned_phrases = ['import', '__']
+    for phrase in banned_phrases:
+        assert phrase not in code_str
+  
+    if gvars is None:
+        gvars = {}
+    if lvars is None:
+        lvars = {}
+    empty_fn = lambda *args, **kwargs: None
+    custom_gvars = merge_dicts([
+        gvars,
+        {'exec': empty_fn, 'eval': empty_fn}
+    ])
+
+    try:
+        exec(code_str, custom_gvars, lvars)
+    except Exception as e:
+        traceback.print_exc()
+    # exec(code_str, custom_gvars, lvars)

md_logger.py

@@ -0,0 +1,16 @@
+class MarkdownLogger:
+
+    def __init__(self):
+        self._log = ''
+
+    def log_text(self, text):
+        self._log += '\n' + text + '\n'
+    
+    def log_code(self, code):
+        self._log += f'\n```python\n{code}\n```\n'
+
+    def clear(self):
+        self._log = ''
+
+    def get_log(self):
+        return self._log

prompts/fgen.py

@@ -0,0 +1,49 @@
+import numpy as np
+from shapely.geometry import *
+from shapely.affinity import *
+
+from env_utils import get_obj_pos, get_obj_names
+from ctrl_utils import put_first_on_second
+
+# define function: total = get_total(xs=numbers).
+def get_total(xs):
+    return np.sum(xs)
+
+# define function: y = eval_line(x, slope, y_intercept=0).
+def eval_line(x, slope, y_intercept):
+    return x * slope + y_intercept
+
+# define function: pt = get_pt_to_the_left(pt, dist).
+def get_pt_to_the_left(pt, dist):
+    return pt + [-dist, 0]
+
+# define function: pt = get_pt_to_the_top(pt, dist).
+def get_pt_to_the_top(pt, dist):
+    return pt + [0, dist]
+
+# define function line = make_line_by_length(length=x).
+def make_line_by_length(length):
+  line = LineString([[0, 0], [length, 0]])
+  return line
+
+# define function: line = make_vertical_line_by_length(length=x).
+def make_vertical_line_by_length(length):
+  line = make_line_by_length(length)
+  vertical_line = rotate(line, 90)
+  return vertical_line
+
+# define function: pt = interpolate_line(line, t=0.5).
+def interpolate_line(line, t):
+  pt = line.interpolate(t, normalized=True)
+  return np.array(pt.coords[0])
+
+# example: scale a line by 2.
+line = make_line_by_length(1)
+new_shape = scale(line, xfact=2, yfact=2)
+
+# example: put object1 on top of object0.
+put_first_on_second('object1', 'object0')
+
+# example: get the position of the first object.
+obj_names = get_obj_names()
+pos_2d = get_obj_pos(obj_names[0])

prompts/parse_obj_name.py

@@ -0,0 +1,59 @@
+import numpy as np
+from env_utils import get_obj_pos, parse_position
+from utils import get_obj_positions_np
+
+objects = ['blue block', 'cyan block', 'purple bowl', 'gray bowl', 'brown bowl', 'pink block', 'purple block']
+# the block closest to the purple bowl.
+block_names = ['blue block', 'cyan block', 'purple block']
+closest_block_idx = get_closest_idx(points=get_obj_positions_np(block_names), point=get_obj_pos('purple bowl'))
+closest_block_name = block_names[closest_block_idx]
+ret_val = closest_block_name
+objects = ['brown bowl', 'banana', 'brown block', 'apple', 'blue bowl', 'blue block']
+# the blocks.
+ret_val = ['brown block', 'blue block']
+objects = ['brown bowl', 'banana', 'brown block', 'apple', 'blue bowl', 'blue block']
+# the brown objects.
+ret_val = ['brown bowl', 'brown block']
+objects = ['brown bowl', 'banana', 'brown block', 'apple', 'blue bowl', 'blue block']
+# a fruit that's not the apple
+fruit_names = ['banana', 'apple']
+for fruit_name in fruit_names:
+    if fruit_name != 'apple':
+        ret_val = fruit_name
+objects = ['blue block', 'cyan block', 'purple bowl', 'brown bowl', 'purple block']
+# blocks above the brown bowl.
+block_names = ['blue block', 'cyan block', 'purple block']
+brown_bowl_pos = get_obj_pos('brown bowl')
+use_block_names = []
+for block_name in block_names:
+    if get_obj_pos(block_name)[1] > brown_bowl_pos[1]:
+        use_block_names.append(block_name)
+ret_val = use_block_names
+objects = ['blue block', 'cyan block', 'purple bowl', 'brown bowl', 'purple block']
+# the blue block.
+ret_val = 'blue block'
+objects = ['blue block', 'cyan block', 'purple bowl', 'brown bowl', 'purple block']
+# the block closest to the bottom right corner.
+corner_pos = parse_position('bottom right corner')
+block_names = ['blue block', 'cyan block', 'purple block']
+closest_block_idx = get_closest_idx(points=get_obj_positions_np(block_names), point=corner_pos)
+closest_block_name = block_names[closest_block_idx]
+ret_val = closest_block_name
+objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
+# the left most block.
+block_names = ['green block', 'brown block', 'blue block']
+left_block_idx = np.argsort(get_obj_positions_np(block_names)[:, 0])[0]
+left_block_name = block_names[left_block_idx]
+ret_val = left_block_name
+objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
+# the bowl on near the top.
+bowl_names = ['brown bowl', 'green bowl', 'blue bowl']
+top_bowl_idx = np.argsort(get_obj_positions_np(bowl_names)[:, 1])[-1]
+top_bowl_name = bowl_names[top_bowl_idx]
+ret_val = top_bowl_name
+objects = ['yellow bowl', 'purple block', 'yellow block', 'purple bowl', 'pink bowl', 'pink block']
+# the third bowl from the right.
+bowl_names = ['yellow bowl', 'purple bowl', 'pink bowl']
+bowl_idx = np.argsort(get_obj_positions_np(bowl_names)[:, 0])[-3]
+bowl_name = bowl_names[bowl_idx]
+ret_val = bowl_name