diff --git a/.gitignore b/.gitignore index a43b8bc..842fc68 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ dataset/DEPTH_image/ dataset/RGB_image/ +# emacs files +*~ \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 4639d20..4966a37 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,11 +2,16 @@ cmake_minimum_required(VERSION 2.8.3) project(robotic_gazebo) -find_package(catkin REQUIRED COMPONENTS rospy) +find_package(catkin REQUIRED COMPONENTS) -catkin_package() - -find_package(roslaunch) +catkin_package( + CATKIN_DEPENDS + roslaunch + rospy + gazebo_ros + edy_dualarm_description + warehouse +) foreach(dir launch src) install(DIRECTORY ${dir}/ diff --git a/README.md b/README.md index c24a2ac..f8ae525 100644 --- a/README.md +++ b/README.md @@ -1,91 +1,80 @@ # simData_imgSaver -ros package for saving simulation images in Gazebo with Kinect +ros package for saving simulation images in Gazebo with Kinect. ### Get the package ``` cd ~/catkin_ws/src/ + +# clone this repo git clone https://github.com/ivalab/simData_imgSaver.git -cd .. -catkin_make -``` -### Clone the simData -``` -cd ~/catkin_ws/src/ -git clone https://github.com/ivalab/simData.git -``` -rename the `/3Dwarehouse/tools` to `/warehouse/models` +# make sure to clone dependencies too +git clone https://github.com/ivalab/simData.git +git clone https://github.com/ivaROS/ivaEdy.git -### Preprocess data -in `utils/move_model_files.m` -modify the `dataRoot` -``` -dataRoot = '/home/YOUR/PATH/TO/catkin_ws/src/simData_v2/warehouse/models'; +cd .. +catkin_make ``` -run `move_model_files.m` in MATLAB -### Copy models -copy all models to `/home/YOURNAME/.gazebo/models/` - -NOTE: -you can add `export GAZEBO_RESOURCE_PATH=$GAZEBO_RESOURCE_PATH:/home/YOUR/PATH/TO/catkin_ws/src/simData_v2/warehouse/` in `~/.bashrc` -then you can might skip this step -but change line 34: -``` -fprintf(fsdf, [' file://models/' FolderName '/mesh/' filename '.dae\n']); -``` +The [simData](https://github.com/ivalab/simData.git) package will generate the relevant model files and expose them to gazebo. ### Open Gazebo + ``` rosrun gazebo_ros gazebo ``` ### Run imgSaver with object -in `visuomotor_grasp_3D_Box.py` -modify `MODEL_DIR` to `/home/YOUR/PATH/TO/catkin_ws/src/simData_v2/warehouse/models` + ``` python visuomotor_grasp_3D_Box.py ``` + the files will be saved to `/dataset` ### Run imgSaver with affordance -in `visuomotor_grasp_3D_Box_affordance.py` -modify `MODEL_DIR` to `/home/YOUR/PATH/TO/catkin_ws/src/simData_v2/warehouse/models` + ``` python visuomotor_grasp_3D_Box_affordance.py ``` -it loads corresponding affordance .dae files with dark table and highlight the affordance with specific color + +it loads corresponding affordance .dae files with dark table and highlight the affordance with specific color the files will be saved to `/dataset` ### Remove shadow effect -When saving affordance images, the light source causes shadow and affects the color. The light source is not loadable in code (at lease in older version of Gazebo). One quick and effective solution is to add multiple light sources in Gazebo with GUI. + +When saving affordance images, the light source causes shadow and affects the color. The light source is not loadable in code (at lease in older version of Gazebo). One quick and effective solution is to add multiple light sources in Gazebo with GUI. -Click on the parallel light source and select a location to place, and click on the icon again to turn it on. Also for tall object such as large mug, it is useful to also put several all-directional light source in front of the object. +Click on the parallel light source and select a location to place, and click on the icon again to turn it on. Also for tall object such as large mug, it is useful to also put several all-directional light source in front of the object. ### Process real data -To generate bonding boxes for UMD dataset, put `simData_imgSaver/utils/boundingBoxGenerator.m` in `YOUR/PATH/TO/UMD_affordance/part-affordance-dataset/` and run the script. + +To generate bonding boxes for UMD dataset, put `simData_imgSaver/utils/boundingBoxGenerator.m` in `YOUR/PATH/TO/UMD_affordance/part-affordance-dataset/` and run the script. We segment the blue lazy Susan table and binarize the output to get the bounding boxes in HSV space. You can modify the value to adjust to your case. + ``` -hsv(:,:,3) > INTENSITY_T; % filter out black color -hsv(:,:,2) > 0.33; % filter out white color +hsv(:,:,3) > INTENSITY_T; % filter out black color +hsv(:,:,2) > 0.33; % filter out white color ``` -Real data with bounding box - -Sim data - -Sim data with bounging box - -Sim data with grasp affordance - -Sim data with hit affordance - + +Real data with bounding box + +Sim data + +Sim data with bounging box + +Sim data with grasp affordance + +Sim data with hit affordance + ### Citation + If you find it helpful for your research, please consider citing: @inproceedings{chu2019learning, @@ -101,6 +90,6 @@ If you find it helpful for your research, please consider citing: month = {April} } - ### Contact + If you encounter any questions, please contact me at fujenchu[at]gatech[dot]edu diff --git a/launch/load_3d_warehouse.launch b/launch/load_3d_warehouse.launch index 0329a39..4762496 100644 --- a/launch/load_3d_warehouse.launch +++ b/launch/load_3d_warehouse.launch @@ -18,17 +18,16 @@ - + - + + args="-urdf -x -1 -y 0 -z 0.0 -model edy_dualarm -param robot_description"/> diff --git a/launch/load_arm.launch b/launch/load_arm.launch index fc96d8c..42e42c1 100644 --- a/launch/load_arm.launch +++ b/launch/load_arm.launch @@ -19,11 +19,11 @@ - + diff --git a/launch/load_hammer.launch b/launch/load_hammer.launch new file mode 100644 index 0000000..61b4b0a --- /dev/null +++ b/launch/load_hammer.launch @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/launch/load_obejct.launch b/launch/load_object.launch similarity index 100% rename from launch/load_obejct.launch rename to launch/load_object.launch diff --git a/models/kinect_ros/model.sdf b/models/kinect_ros/model.sdf index cd75ef9..4c26eee 100755 --- a/models/kinect_ros/model.sdf +++ b/models/kinect_ros/model.sdf @@ -1,8 +1,10 @@ - - + + - 0.5 -0.55 1.05 0 0.8722 1.57 + false + 0.5 -0.55 1.05 0 0.8722 1.57 + 0.1 @@ -21,9 +23,13 @@ + + 20 + 1 + 1 1.047198 @@ -40,7 +46,7 @@ 0.2 true + will control the frame rate. --> 0.0 camera_ir /camera/depth/image_raw @@ -62,7 +68,7 @@ 0 - + true - + \ No newline at end of file diff --git a/models/kinect_ros/model.sdf~ b/models/kinect_ros/model.sdf~ deleted file mode 100755 index bd45fb5..0000000 --- a/models/kinect_ros/model.sdf~ +++ /dev/null @@ -1,68 +0,0 @@ - - - - 0.5 -1.05 1.05 0 0.8722 1.57 - - - 0.1 - - - - - - 0.073000 0.276000 0.072000 - - - - - - - model://kinect/meshes/kinect.dae - - - - - - 20 - - 1.047198 - - 640 - 480 - R8G8B8 - - - 0.05 - 3 - - - - 0.2 - true - - 0.0 - camera_ir - /camera/depth/image_raw - /camera/depth/camera_info - /camera/depth/image_raw - /camera/depth/camera_info - /camera/depth/points - camera_link - 0.05 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - true - - diff --git a/models/kinect_ros/model_bak.sdf b/models/kinect_ros/model_bak.sdf index 4d2226a..0961d1c 100644 --- a/models/kinect_ros/model_bak.sdf +++ b/models/kinect_ros/model_bak.sdf @@ -1,4 +1,4 @@ - + @@ -67,4 +67,4 @@ - + \ No newline at end of file diff --git a/models/table/model-1_2.sdf~ b/models/table/model-1_2.sdf~ deleted file mode 100644 index fd6a6b8..0000000 --- a/models/table/model-1_2.sdf~ +++ /dev/null @@ -1,134 +0,0 @@ - - - - true - - - 0 0 0.58 0 0 0 - - - 1.4 0.8 0.04 - - - - - - 0.6 - 0.6 - - - - - - 0 0 0.58 0 0 0 - - - 1.4 0.8 0.04 - - - - 0 0 0 1 - 0 0 0 1 - 0.1 0.1 0.1 1 - 0 0 0 0 - - - - 0.68 0.38 0.28 0 0 0 - - - 0.02 - 0.56 - - - - - 0.68 0.38 0.28 0 0 0 - - - 0.02 - 0.56 - - - - - - - - 0.68 -0.38 0.28 0 0 0 - - - 0.02 - 0.56 - - - - - 0.68 -0.38 0.28 0 0 0 - - - 0.02 - 0.56 - - - - - - - - -0.68 -0.38 0.28 0 0 0 - - - 0.02 - 0.56 - - - - - -0.68 -0.38 0.28 0 0 0 - - - 0.02 - 0.56 - - - - - - - - -0.68 0.38 0.28 0 0 0 - - - 0.02 - 0.56 - - - - - -0.68 0.38 0.28 0 0 0 - - - 0.02 - 0.56 - - - - - - - - - diff --git a/package.xml b/package.xml index 84cf8af..6028635 100644 --- a/package.xml +++ b/package.xml @@ -7,11 +7,29 @@ me BSD + catkin roslaunch rospy + gazebo_ros + edy_dualarm_description + warehouse + + roslaunch + rospy + gazebo + gazebo_ros + gazebo_ros_pkgs robot_state_publisher joint_state_publisher rviz - gazebo - + edy_dualarm_description + warehouse + + + + + \ No newline at end of file diff --git a/src/visuomotor_dataset_collector.py b/src/visuomotor_dataset_collector.py index a63ebca..6242335 100644 --- a/src/visuomotor_dataset_collector.py +++ b/src/visuomotor_dataset_collector.py @@ -44,7 +44,7 @@ gripper_open_value = -0.7 PI = 3.1415 STEPS_ARM_POSE = 100 -STEPS_OBEJCT_NUM = 90 +STEPS_object_NUM = 90 STEPS_OBJECT_RANDOM_POSE = 1 STEPS_OBJECT_CLOSE_POSE = 1 STEPS_CAM_POSE = 1 @@ -77,7 +77,7 @@ def main(): ### start node - rospy.init_node('gazebo_arm_obejct') + rospy.init_node('gazebo_arm_object') #initialize publishers for 9 motors pub_1 = rospy.Publisher('/finalarm_released_v0/joint_1_position_controller/command', Float64, queue_size=5) @@ -141,11 +141,11 @@ def main(): # loop for different object model # ################################### count_object_num = 0 - while count_object_num < STEPS_OBEJCT_NUM: + while count_object_num < STEPS_object_NUM: ### loop for diff random object pose - count_obejct_random_pose = 0 + count_object_random_pose = 0 count_object_close_pose = 0 - while count_obejct_random_pose < STEPS_OBJECT_RANDOM_POSE: + while count_object_random_pose < STEPS_OBJECT_RANDOM_POSE: #generate new pose for object pose_obj = random_pose_object(req.pose_stamped.pose.position.x, req.pose_stamped.pose.position.y) #pose_obj = Pose(Point(0.40,0.2,0.63),Quaternion(1, 0, 0, 0)) @@ -274,7 +274,7 @@ def main(): count_cam_pose += 1 ###increment count for obj pose - count_obejct_random_pose += 1 + count_object_random_pose += 1 ################################### # loop for diff close object pose # @@ -415,7 +415,7 @@ def main(): delete_model("object") ###wait the object model to be deleted - while "obejct" in world_properties.model_names: + while "object" in world_properties.model_names: world_properties = get_world_properties() ###increment count for obj no diff --git a/src/visuomotor_grasp_3D_Box.py b/src/visuomotor_grasp_3D_Box.py index 0b6217b..f3494c2 100755 --- a/src/visuomotor_grasp_3D_Box.py +++ b/src/visuomotor_grasp_3D_Box.py @@ -15,6 +15,7 @@ import tf import numpy as np from cv_bridge import CvBridge, CvBridgeError +from pathlib import Path #globals object_pose = Pose(Point(0,0,0), Quaternion(1,0,0,0)) @@ -22,11 +23,11 @@ PI = 3.1415 STEPS_OBJECT_POSE = 100 STEPS_CAM_POSE = 10 -HOME = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..')) +HOME = Path(__file__ ).parent.parent.as_posix() DEBUG = True RGB_IMG_DIR = HOME + "/dataset/RGB_image" DEPTH_IMG_DIR = HOME + "/dataset/DEPTH_image" -MODEL_DIR = "/home/fujenchu/projects/robotArm/catkin_ws/src/simData/warehouse/models" +MODEL_DIR = HOME + "/../simData/models" ### ros service proxies print("Waiting for gazebo servies, be sure to spawn gazebo_ros with rosrun gazebo_ros gazebo or rosrun gazebo_ros gzserver") @@ -48,7 +49,7 @@ def main(): ### start node - rospy.init_node('gazebo_arm_obejct') + rospy.init_node('gazebo_arm_object') world_properties = get_world_properties() if DEBUG: print("World properties: "); print(world_properties); print("\n") diff --git a/src/visuomotor_grasp_3D_Box_affordance.py b/src/visuomotor_grasp_3D_Box_affordance.py index 71f8fd9..701fe25 100755 --- a/src/visuomotor_grasp_3D_Box_affordance.py +++ b/src/visuomotor_grasp_3D_Box_affordance.py @@ -16,17 +16,19 @@ import numpy as np from cv_bridge import CvBridge, CvBridgeError import math +from pathlib import Path #globals object_pose = Pose(Point(0,0,0), Quaternion(1,0,0,0)) gripper_open_value = -0.7 PI = 3.1415 STEPS_OBJECT_POSE = 400 -HOME = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..')) +HOME = Path(__file__ ).parent.parent.as_posix() DEBUG = True RGB_IMG_DIR = HOME + "/dataset/RGB_image" DEPTH_IMG_DIR = HOME + "/dataset/DEPTH_image" -MODEL_DIR = "/home/fujenchu/projects/robotArm/catkin_ws/src/simData/warehouse/models" +# OR output from `rospack plugins --attrib=gazebo_model_path gazebo_ros ` +MODEL_DIR = HOME + "/../simData/models" ### ros service proxies print("Waiting for gazebo servies, be sure to spawn gazebo_ros with rosrun gazebo_ros gazebo or rosrun gazebo_ros gzserver") @@ -69,7 +71,7 @@ def main(): ### start node - rospy.init_node('gazebo_arm_obejct') + rospy.init_node('gazebo_arm_object') world_properties = get_world_properties() if DEBUG: print("World properties: "); print(world_properties); print("\n") diff --git a/utils/move_model_files.m~ b/utils/move_model_files.m~ deleted file mode 100644 index 066b9e3..0000000 --- a/utils/move_model_files.m~ +++ /dev/null @@ -1,55 +0,0 @@ -%move files from one directory to another directory - -dataRoot = '/home/fujenchu/projects/robotArm/catkin_ws/src/simData/warehouse/models'; -dataDir= dir(dataRoot); - -for idx_folder = 3:length(dataDir) - FolderName = dataDir(idx_folder).name; - fileRoot = [dataRoot '/' FolderName '/' FolderName '_origin.dae']; - [pathstr,filename] = fileparts(fileRoot); - mkdir([dataRoot '/' FolderName '/mesh']); - copyfile([dataRoot '/' FolderName '/*.dae'], [dataRoot '/' FolderName '/mesh']); - mkdir([dataRoot '/' FolderName '/sdf']); - fsdf = fopen([dataRoot '/' FolderName '/sdf/description.sdf'],'wt'); - fprintf(fsdf, '\n'); %write ' to a file needs '' in the string - fprintf(fsdf, '\n'); - fprintf(fsdf, ['\n']); - fprintf(fsdf, ' true\n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' true\n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' 0.1 0.1 0.1\n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, [' model://' FolderName '/mesh/' filename '.dae\n']); - fprintf(fsdf, ' 1 1 1\n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ' \n'); - fprintf(fsdf, ''); - fclose(fsdf); - allFiles = dir(pathstr); - dirFlags = [allFiles.isdir]; - subFolder = allFiles(dirFlags); - idx = 1; - while idx <= length(subFolders) - if strcmp(subFolders(idx).name,'mesh') || strcmp(subFolders(idx).name,'sdf') - subFolders(idx)=[]; - else - idx = idx +1; - end - end - -end \ No newline at end of file