Support instancing in MCP select_assets and read_asset

src/tydra/mcp-context.hh

@@ -20,6 +20,18 @@ struct Image
   std::string data; // based64 encoded image
 };
 
+struct AssetSelection
+{
+  std::string asset_name;
+
+  // Instance and transform parameters
+  int instance_id = 0; // Instance ID for the asset
+  std::array<float, 3> position = {0.0f, 0.0f, 0.0f}; // x, y, z
+  std::array<float, 3> scale = {1.0f, 1.0f, 1.0f}; // x, y, z
+  std::array<float, 3> rotation = {0.0f, 0.0f, 0.0f}; // x, y, z Euler angles in degrees
+
+};
+
 // Generic Asset(USD, textures, etc.)
 struct MCPAsset
 {
@@ -29,11 +41,11 @@ struct MCPAsset
   Image preview; // preview image of the asset(optional)
   std::string uuid;
   
-  // Instance and transform parameters
-  int instance_id = 0; // Instance ID for the asset
-  std::array<float, 3> position = {0.0f, 0.0f, 0.0f}; // x, y, z
-  std::array<float, 3> scale = {1.0f, 1.0f, 1.0f}; // x, y, z
-  std::array<float, 3> rotation = {0.0f, 0.0f, 0.0f}; // x, y, z Euler angles in degrees
+  
+  // Geometry and bounding box parameters
+  std::array<float, 3> pivot_position = {0.0f, 0.0f, 0.0f}; // pivot point for rotation and scaling
+  std::array<float, 3> bmin = {-1.0f, -1.0f, -1.0f}; // bounding box minimum
+  std::array<float, 3> bmax = {1.0f, 1.0f, 1.0f}; // bounding box maximum
 };
 
 struct USDLayer
@@ -61,7 +73,7 @@ struct Context
   // key = name
   std::unordered_map<std::string, MCPAsset> assets;
 
-  std::vector<std::string> selected_assets;
+  std::vector<AssetSelection> selected_assets;
 
   // key = name
   std::unordered_map<std::string, Screenshot> screenshots;

src/tydra/mcp-tools.cc

@@ -238,23 +238,37 @@ bool ReadAsset(Context &ctx, const nlohmann::json &args, nlohmann::json &result,
   }
 
   std::string name = args["name"];
+  int instance_id = -1;
+  if (args.contains("instance_id")) {
+    instance_id = args["instance_id"];
+  }
 
-  if (!ctx.assets.count(name)) {
+  AssetSelection asset_selection;
+  bool found_selection = false;
+
+  // simple linear search
+  for (const auto &selection : ctx.selected_assets) {
+    if (selection.asset_name == name && (instance_id == -1 || selection.instance_id == instance_id)) {
+
+      asset_selection = selection;
+      found_selection = true;
+
+      DCOUT("Found matching asset selection: " << selection.asset_name);
+      break;
+    }
+  } 
+
+  if (!found_selection) {
+    err = "Asset selection not found for name: " + name;
+    return false;
+  }
+
+  if (!ctx.assets.count(asset_selection.asset_name)) {
     err = "Asset not found: " + name;
     return false;
   }
-  const MCPAsset &asset = ctx.assets.at(name);
 
-  // Check instance_id if provided
-  if (args.contains("instance_id") && args["instance_id"].is_number_integer()) {
-    int requested_instance_id = args["instance_id"];
-    if (asset.instance_id != requested_instance_id) {
-      err = "Asset '" + name + "' instance_id mismatch. Expected: " + 
-            std::to_string(requested_instance_id) + ", Found: " + 
-            std::to_string(asset.instance_id);
-      return false;
-    }
-  }
+  const MCPAsset &asset = ctx.assets.at(asset_selection.asset_name);
 
   // Create JSON response with asset data and transform information
   nlohmann::json asset_data;
@@ -264,10 +278,15 @@ bool ReadAsset(Context &ctx, const nlohmann::json &args, nlohmann::json &result,
   asset_data["uuid"] = asset.uuid;
   
   // Add instance and transform parameters
-  asset_data["instance_id"] = asset.instance_id;
-  asset_data["position"] = nlohmann::json::array({asset.position[0], asset.position[1], asset.position[2]});
-  asset_data["scale"] = nlohmann::json::array({asset.scale[0], asset.scale[1], asset.scale[2]});
-  asset_data["rotation"] = nlohmann::json::array({asset.rotation[0], asset.rotation[1], asset.rotation[2]});
+  asset_data["instance_id"] = asset_selection.instance_id;
+  asset_data["position"] = nlohmann::json::array({asset_selection.position[0], asset_selection.position[1], asset_selection.position[2]});
+  asset_data["scale"] = nlohmann::json::array({asset_selection.scale[0], asset_selection.scale[1], asset_selection.scale[2]});
+  asset_data["rotation"] = nlohmann::json::array({asset_selection.rotation[0], asset_selection.rotation[1], asset_selection.rotation[2]});
+
+  // Add geometry and bounding box parameters
+  asset_data["pivot_position"] = nlohmann::json::array({asset.pivot_position[0], asset.pivot_position[1], asset.pivot_position[2]});
+  asset_data["bmin"] = nlohmann::json::array({asset.bmin[0], asset.bmin[1], asset.bmin[2]});
+  asset_data["bmax"] = nlohmann::json::array({asset.bmax[0], asset.bmax[1], asset.bmax[2]});
 
   nlohmann::json content;
   content["type"] = "text";
@@ -322,6 +341,25 @@ bool StoreAsset(Context &ctx, const nlohmann::json &args,
     }
   }
 
+  // Handle geometry and bounding box parameters if provided
+  if (args.contains("pivot_position") && args["pivot_position"].is_array() && args["pivot_position"].size() == 3) {
+    asset.pivot_position[0] = args["pivot_position"][0];
+    asset.pivot_position[1] = args["pivot_position"][1];
+    asset.pivot_position[2] = args["pivot_position"][2];
+  }
+
+  if (args.contains("bmin") && args["bmin"].is_array() && args["bmin"].size() == 3) {
+    asset.bmin[0] = args["bmin"][0];
+    asset.bmin[1] = args["bmin"][1];
+    asset.bmin[2] = args["bmin"][2];
+  }
+
+  if (args.contains("bmax") && args["bmax"].is_array() && args["bmax"].size() == 3) {
+    asset.bmax[0] = args["bmax"][0];
+    asset.bmax[1] = args["bmax"][1];
+    asset.bmax[2] = args["bmax"][2];
+  }
+
   ctx.assets.emplace(name, std::move(asset));
 
   nlohmann::json content;
@@ -577,6 +615,11 @@ bool GetAssetDescription(Context &ctx, const nlohmann::json &args,
   asset_info["asset_name"] = asset.name;
   asset_info["description"] = asset.description;
   asset_info["uuid"] = asset.uuid;
+  
+  // Add geometry and bounding box parameters
+  asset_info["pivot_position"] = nlohmann::json::array({asset.pivot_position[0], asset.pivot_position[1], asset.pivot_position[2]});
+  asset_info["bmin"] = nlohmann::json::array({asset.bmin[0], asset.bmin[1], asset.bmin[2]});
+  asset_info["bmax"] = nlohmann::json::array({asset.bmax[0], asset.bmax[1], asset.bmax[2]});
 
   // Add preview data if available and requested
   if (include_preview && !asset.preview.data.empty()) {
@@ -618,6 +661,11 @@ bool GetAllAssetDescriptions(Context &ctx, const nlohmann::json &args,
     asset_info["asset_name"] = it.second.name;
     asset_info["description"] = it.second.description;
     asset_info["uuid"] = it.second.uuid;
+    
+    // Add geometry and bounding box parameters
+    asset_info["pivot_position"] = nlohmann::json::array({it.second.pivot_position[0], it.second.pivot_position[1], it.second.pivot_position[2]});
+    asset_info["bmin"] = nlohmann::json::array({it.second.bmin[0], it.second.bmin[1], it.second.bmin[2]});
+    asset_info["bmax"] = nlohmann::json::array({it.second.bmax[0], it.second.bmax[1], it.second.bmax[2]});
 
     if (include_preview) {
       // Add preview data if available
@@ -677,7 +725,7 @@ bool ToUSDA(Context &ctx, const nlohmann::json &args, nlohmann::json &result,
 
 bool SelectAssets(Context &ctx, const nlohmann::json &args,
                   nlohmann::json &result, std::string &err) {
-  DCOUT("args " << args);
+  std::cout << "select_assets"  << args << "\n";
   if (!args.contains("assets")) {
     DCOUT("assets param not found");
     err = "`assets` param not found.";
@@ -707,8 +755,8 @@ bool SelectAssets(Context &ctx, const nlohmann::json &args,
     std::string name = asset_obj["name"];
     
     if (!ctx.assets.count(name)) {
-      DCOUT("Asset not found: " << name);
-      continue; // Skip assets that don't exist
+      err = "Asset not found" + name;
+      return false;
     }
     
     // Default instance and transform parameters
@@ -717,6 +765,11 @@ bool SelectAssets(Context &ctx, const nlohmann::json &args,
     std::array<float, 3> scale = {1.0f, 1.0f, 1.0f};
     std::array<float, 3> rotation = {0.0f, 0.0f, 0.0f};
     
+    // Default geometry and bounding box parameters
+    std::array<float, 3> pivot_position = {0.0f, 0.0f, 0.0f};
+    std::array<float, 3> bmin = {-1.0f, -1.0f, -1.0f};
+    std::array<float, 3> bmax = {1.0f, 1.0f, 1.0f};
+    
     // Parse instance_id
     if (asset_obj.contains("instance_id") && asset_obj["instance_id"].is_number_integer()) {
       instance_id = asset_obj["instance_id"];
@@ -741,17 +794,44 @@ bool SelectAssets(Context &ctx, const nlohmann::json &args,
       rotation[2] = asset_obj["rotation"][2];
     }
     
-    // Update the asset with its individual transform parameters
-    ctx.assets[name].instance_id = instance_id;
-    ctx.assets[name].position = position;
-    ctx.assets[name].scale = scale;
-    ctx.assets[name].rotation = rotation;
-    ctx.selected_assets.push_back(name);
+    // Parse geometry and bounding box parameters
+    if (asset_obj.contains("pivot_position") && asset_obj["pivot_position"].is_array() && asset_obj["pivot_position"].size() == 3) {
+      pivot_position[0] = asset_obj["pivot_position"][0];
+      pivot_position[1] = asset_obj["pivot_position"][1];
+      pivot_position[2] = asset_obj["pivot_position"][2];
+    }
     
-    DCOUT("Selected asset '" << name << "' (instance_id: " << instance_id << ") with transform - Position: [" 
+    if (asset_obj.contains("bmin") && asset_obj["bmin"].is_array() && asset_obj["bmin"].size() == 3) {
+      bmin[0] = asset_obj["bmin"][0];
+      bmin[1] = asset_obj["bmin"][1];
+      bmin[2] = asset_obj["bmin"][2];
+    }
+    
+    if (asset_obj.contains("bmax") && asset_obj["bmax"].is_array() && asset_obj["bmax"].size() == 3) {
+      bmax[0] = asset_obj["bmax"][0];
+      bmax[1] = asset_obj["bmax"][1];
+      bmax[2] = asset_obj["bmax"][2];
+    }
+    
+    // Update the asset with its individual transform parameters
+    AssetSelection selection;
+    selection.asset_name = name;
+    selection.instance_id = instance_id;
+    selection.position = position;
+    selection.scale = scale;
+    selection.rotation = rotation;
+    //selection.pivot_position = pivot_position;
+    //selection.bmin = bmin;
+    //selection.bmax = bmax;
+    ctx.selected_assets.push_back(selection);
+    
+    std::cout << "Selected asset '" << name << "' (instance_id: " << instance_id << ") with transform - Position: [" 
           << position[0] << ", " << position[1] << ", " << position[2] << "], "
           << "Scale: [" << scale[0] << ", " << scale[1] << ", " << scale[2] << "], "
-          << "Rotation: [" << rotation[0] << ", " << rotation[1] << ", " << rotation[2] << "]");
+          << "Rotation: [" << rotation[0] << ", " << rotation[1] << ", " << rotation[2] << "], "
+          << "Pivot: [" << pivot_position[0] << ", " << pivot_position[1] << ", " << pivot_position[2] << "], "
+          << "BMin: [" << bmin[0] << ", " << bmin[1] << ", " << bmin[2] << "], "
+          << "BMax: [" << bmax[0] << ", " << bmax[1] << ", " << bmax[2] << "]";
   }
 
   result["content"] = nlohmann::json::array();
@@ -766,11 +846,11 @@ bool GetSelectedAssets(Context &ctx, const nlohmann::json &args,
   DCOUT("args " << args);
 
   result["content"] = nlohmann::json::array();
-  for (const auto &name : ctx.selected_assets) {
+  for (const auto &selection : ctx.selected_assets) {
     // Create JSON object with name and instance_id
     nlohmann::json asset_info;
-    asset_info["name"] = name;
-    asset_info["instance_id"] = ctx.assets.at(name).instance_id;
+    asset_info["name"] = selection.asset_name;
+    asset_info["instance_id"] = selection.instance_id;
     
     nlohmann::json content;
     content["type"] = "text";
@@ -952,7 +1032,7 @@ bool GetToolsList(Context &ctx, nlohmann::json &result) {
     nlohmann::json j;
     j["name"] = "store_asset";
     j["description"] =
-        "Store asset(e.g. USD, texture) with optional preview image. `data` is "
+        "Store asset(e.g. USD, texture) with optional preview image and geometry parameters (pivot_position, bmin, bmax). `data` is "
         "base64 encoded string.";
 
     nlohmann::json schema;
@@ -978,6 +1058,23 @@ bool GetToolsList(Context &ctx, nlohmann::json &result) {
 
     schema["properties"]["preview"] = previewSchema;  // optional
 
+    // Add geometry and bounding box parameters
+    schema["properties"]["pivot_position"] = {{"type", "array"},
+                                              {"items", {"type", "number"}},
+                                              {"minItems", 3},
+                                              {"maxItems", 3},
+                                              {"description", "Pivot position as [x, y, z] for rotation and scaling"}};
+    schema["properties"]["bmin"] = {{"type", "array"},
+                                    {"items", {"type", "number"}},
+                                    {"minItems", 3},
+                                    {"maxItems", 3},
+                                    {"description", "Bounding box minimum as [x, y, z]"}};
+    schema["properties"]["bmax"] = {{"type", "array"},
+                                    {"items", {"type", "number"}},
+                                    {"minItems", 3},
+                                    {"maxItems", 3},
+                                    {"description", "Bounding box maximum as [x, y, z]"}};
+
     schema["required"] = nlohmann::json::array({"data", "name"});
 
     j["inputSchema"] = schema;
@@ -1102,6 +1199,21 @@ bool GetToolsList(Context &ctx, nlohmann::json &result) {
                                              {"minItems", 3},
                                              {"maxItems", 3},
                                              {"description", "Rotation as [x, y, z] in degrees"}};
+    assetSchema["properties"]["pivot_position"] = {{"type", "array"},
+                                                   {"items", {"type", "number"}},
+                                                   {"minItems", 3},
+                                                   {"maxItems", 3},
+                                                   {"description", "Pivot position as [x, y, z] for rotation and scaling"}};
+    assetSchema["properties"]["bmin"] = {{"type", "array"},
+                                         {"items", {"type", "number"}},
+                                         {"minItems", 3},
+                                         {"maxItems", 3},
+                                         {"description", "Bounding box minimum as [x, y, z]"}};
+    assetSchema["properties"]["bmax"] = {{"type", "array"},
+                                         {"items", {"type", "number"}},
+                                         {"minItems", 3},
+                                         {"maxItems", 3},
+                                         {"description", "Bounding box maximum as [x, y, z]"}};
     assetSchema["required"] = nlohmann::json::array({"name"});
 
     schema["properties"]["assets"] = {{"type", "array"}, {"items", assetSchema}};

web/demo/mcp-sample.js

@@ -653,19 +653,24 @@ async function connectMCPServer() {
   params.mcpServerConnected = ui_state['mcpServerConnected']; // Update GUI parameter
 }
 
-async function getAsset(name) {
+async function getAsset(asset_info) {
   const client = ui_state['mcpClient'];
   if (!client) {
     console.error('MCP client is not connected');
     return;
   }
 
+  let args = {};
+  args.name = asset_info.name;
+  if (asset_info.instance_id) {
+    args.instance_id = asset_info.instance_id;
+  } 
+  console.log('args:', args);
+
   try {
     const response = await client.callTool({
       name: 'read_asset',
-      arguments: {
-        name: name
-      }
+      arguments: args
     });
     console.log('Asset retrieved:', response);
     
@@ -685,6 +690,7 @@ async function getAsset(name) {
       position: assetInfo.position || [0, 0, 0],
       scale: assetInfo.scale || [1, 1, 1],
       rotation: assetInfo.rotation || [0, 0, 0] // XYZ angles in degrees
+      ,
     };
   } catch (error) {
     console.error('Error retrieving asset:', error);
@@ -807,16 +813,16 @@ async function reloadScenes(loader, asset_names) {
   var assetTransforms = [] // Store transform info for each asset
 
   var usd_scenes = [];
-  for (const asset_name of asset_names) {
-    console.log('Loading asset:', asset_name);
+  for (const asset_jsoninfo of asset_names) {
+    console.log('Loading asset:', asset_jsoninfo);
 
-    const assetInfo = await getAsset(asset_name);
+    const assetInfo = await getAsset(JSON.parse(asset_jsoninfo));
     if (!assetInfo) {
-      console.error('Failed to load asset:', asset_name);
+      console.error('Failed to load asset:', asset_jsoninfo);
       continue;
     }
-    
-    console.log('Asset info for', asset_name, ':', assetInfo);
+
+    console.log('Asset info for', asset_jsoninfo, ':', assetInfo);
 
     const usd_scene = await loader.loadAsync(assetInfo.dataUri);
     console.log('Loaded USD scene:', usd_scene);

web/mcp-server/scripts/generate_asset_description_json.py

@@ -25,6 +25,15 @@ for f in files:
     print(basename)
 
     j = json.loads(open(in_json_file).read())
+
+    # optional meta
+    in_metajson_file = os.path.splitext(f)[0] + "-meta.json"
+    print(in_metajson_file)
+
+    if os.path.exists(in_metajson_file):
+        meta_j = json.loads(open(in_metajson_file).read())
+        j.update(meta_j)
+
     js[basename] = j
 
 out_j = json.dumps(js)

web/mcp-server/setup-asset.js

@@ -63,9 +63,18 @@ for (const [key, value] of Object.entries(descriptions)) {
   const filename = value.usd_filename;
   const description = value.description;
   const preview = value.screenshot_filename;
+  
+  // Read geometry parameters from value, with defaults if not specified
+  const pivot_position = value.pivot_position || [0.0, 0.0, 0.0];
+  const bmin = value.bmin || [-1.0, -1.0, -1.0];
+  const bmax = value.bmax || [1.0, 1.0, 1.0];
+  
   assert(filename)
   assert(description)
   console.log(`filename: ${filename}, desc: ${description}`);
+  console.log(`pivot_position: [${pivot_position.join(', ')}]`);
+  console.log(`bmin: [${bmin.join(', ')}]`);
+  console.log(`bmax: [${bmax.join(', ')}]`);
   if (!preview) {
     console.warn(`No preview image specified for ${filename}`);
   } else {
@@ -80,7 +89,10 @@ for (const [key, value] of Object.entries(descriptions)) {
   let args = {
     "name": filename,
     "data": base64data,
-    "description": description
+    "description": description,
+    "pivot_position": pivot_position,
+    "bmin": bmin,
+    "bmax": bmax
   };
 
   if (preview) {