Isaac GR00T N1.7 파인튜닝 따라 하기: LeRobot 형식 데이터 준비부터 오픈루프 평가까지

sudo apt install git-lfs && git lfs install
git clone --recurse-submodules https://github.com/NVIDIA/Isaac-GR00T
cd Isaac-GR00T

curl -LsSf https://astral.sh/uv/install.sh | sh
sudo apt-get update && sudo apt-get install -y ffmpeg

uv sync --python 3.12
uv run python -c "import gr00t; print('GR00T installed successfully')"
uv run huggingface-cli login   # 또는 export HF_TOKEN=<토큰>

cd scripts/lerobot_conversion
uv venv && source .venv/bin/activate
uv pip install -e . --verbose
python convert_v3_to_v2.py --repo-id <내_계정>/<데이터셋_이름>

meta/
  info.json
  episodes.jsonl
  tasks.jsonl
  modality.json        # GR00T 전용
data/chunk-000/episode_000000.parquet
videos/chunk-000/observation.images.front/episode_000000.mp4
videos/chunk-000/observation.images.wrist/episode_000000.mp4

{
  "state":  { "single_arm": {"start": 0, "end": 5}, "gripper": {"start": 5, "end": 6} },
  "action": { "single_arm": {"start": 0, "end": 5}, "gripper": {"start": 5, "end": 6} },
  "video": {
    "front": {"original_key": "observation.images.front"},
    "wrist": {"original_key": "observation.images.wrist"}
  },
  "annotation": { "human.task_description": {"original_key": "task_index"} }
}

from gr00t.configs.data.embodiment_configs import register_modality_config
from gr00t.data.embodiment_tags import EmbodimentTag
from gr00t.data.types import (
    ActionConfig, ActionFormat, ActionRepresentation, ActionType, ModalityConfig,
)

so100_config = {
    "video": ModalityConfig(delta_indices=[0], modality_keys=["front", "wrist"]),
    "state": ModalityConfig(delta_indices=[0], modality_keys=["single_arm", "gripper"]),
    "action": ModalityConfig(
        delta_indices=list(range(0, 16)),  # 16스텝 예측
        modality_keys=["single_arm", "gripper"],
        action_configs=[
            ActionConfig(rep=ActionRepresentation.RELATIVE, type=ActionType.NON_EEF,
                         format=ActionFormat.DEFAULT),   # 팔: 현재 상태 대비 상대값
            ActionConfig(rep=ActionRepresentation.ABSOLUTE, type=ActionType.NON_EEF,
                         format=ActionFormat.DEFAULT),   # 그리퍼: 절대 목표값
        ],
    ),
    "language": ModalityConfig(delta_indices=[0],
                               modality_keys=["annotation.human.task_description"]),
}

register_modality_config(so100_config, embodiment_tag=EmbodimentTag.NEW_EMBODIMENT)

CUDA_VISIBLE_DEVICES=0 uv run python \
    gr00t/experiment/launch_finetune.py \
    --base-model-path nvidia/GR00T-N1.7-3B \
    --dataset-path demo_data/cube_to_bowl_5 \
    --embodiment-tag NEW_EMBODIMENT \
    --modality-config-path examples/SO100/so100_config.py \
    --num-gpus 1 \
    --output-dir /tmp/so100 \
    --save-steps 500 \
    --max-steps 2000 \
    --global-batch-size 32 \
    --dataloader-num-workers 4

uv run python gr00t/eval/open_loop_eval.py \
    --dataset-path ./demo_data/cube_to_bowl_5 \
    --embodiment-tag NEW_EMBODIMENT \
    --model-path /tmp/so100/checkpoint-2000 \
    --traj-ids 0 \
    --execution-horizon 16 \
    --steps 400 \
    --modality-keys single_arm gripper \
    --save-plot-path ./eval_plots

uv run python gr00t/eval/run_gr00t_server.py \
    --embodiment-tag NEW_EMBODIMENT \
    --model-path /tmp/so100/checkpoint-2000 \
    --device cuda:0 --host 0.0.0.0 --port 5555

from gr00t.policy.server_client import PolicyClient

policy = PolicyClient(host="localhost", port=5555, timeout_ms=15000, strict=False)
if not policy.ping():
    raise RuntimeError("Cannot connect to policy server!")

observation = {
    "video": {"front": front_frames, "wrist": wrist_frames},  # (1, T, H, W, 3) uint8
    "state": {"single_arm": arm_state, "gripper": grip_state},  # (1, T, D) float32
    "language": {"task": [["put the cube into the yellow bowl"]]},
}
action, info = policy.get_action(observation)
next_arm = action["single_arm"][:, 0, :]  # (B, T, D) 중 첫 스텝