Add screenshot save/read tool.

src/tydra/mcp-context.hh

@@ -19,6 +19,13 @@ struct USDLayer
   Layer layer;
 };
 
+struct Screenshot
+{
+  std::string uuid;
+  std::string mimeType;
+  std::string data; // base64 encoded image data.
+};
+
 struct Context
 {
 
@@ -28,6 +35,9 @@ struct Context
 
   // key = URI, value = UUID
   std::unordered_map<std::string, std::string> resources;
+
+  // key = name
+  std::unordered_map<std::string, Screenshot> screenshots;
 };
 
 } // namespace mcp

src/tydra/mcp-tools.cc

@@ -210,6 +210,97 @@ bool ListPrimSpecs(Context &ctx, const nlohmann::json &args, nlohmann::json &res
   return true;
 }
 
+bool ListScreenshots(Context &ctx, const nlohmann::json &args, nlohmann::json &result, std::string &err) {
+  (void)args;
+  (void)err;
+
+  result["content"] = nlohmann::json::array();
+
+  for (const auto &it : ctx.screenshots) {
+    nlohmann::json content;
+    content["type"] = "text";
+    content["text"] = it.first; // name
+    result["content"].push_back(content);
+  }
+  
+  return true;
+}
+
+bool SaveScreenshot(Context &ctx, const nlohmann::json &args, nlohmann::json &result, std::string &err) {
+  DCOUT("args " << args);
+  if (!args.contains("name")) {
+    DCOUT("name param not found");
+    err = "`name` param not found.\n";
+    return false; 
+  }
+  if (!args.contains("data")) {
+    DCOUT("data param not found");
+    err = "`data` param not found.\n";
+    return false; 
+  }
+  if (!args.contains("mimeType")) {
+    DCOUT("mimeType param not found");
+    err = "`mimeType` param not found.\n";
+    return false; 
+  }
+
+  std::string name = args["name"];
+  std::string data = args["data"];
+  std::string mimeType = args["mimeType"];
+
+  Screenshot screenshot;
+  screenshot.uuid = UUIDGenerator::generateUUID();
+  screenshot.data = data;
+  screenshot.mimeType = mimeType;
+
+  ctx.screenshots[name] = screenshot;
+
+  result["content"] = nlohmann::json::array();
+
+  nlohmann::json content;
+  content["type"] = "text";
+  content["text"] = screenshot.uuid;
+  result["content"].push_back(content);
+  
+  return true;
+}
+
+bool ReadScreenshot(Context &ctx, const nlohmann::json &args, nlohmann::json &result, std::string &err) {
+  DCOUT("args " << args);
+  if (!args.contains("name")) {
+    DCOUT("name param not found");
+    err = "`name` param not found.\n";
+    return false; 
+  }
+
+  std::string name = args["name"];
+
+  if (!ctx.screenshots.count(name)) {
+    DCOUT("Screenshot not found: " << name);
+    err = "Screenshot not found: " + name;
+    return false;
+  }
+
+  const auto &screenshot = ctx.screenshots.at(name);
+
+  result["content"] = nlohmann::json::array();
+
+  nlohmann::json content;
+  content["type"] = "image";
+  content["data"] = screenshot.data; // base64-encoded-data
+  content["mimeType"] = screenshot.mimeType;
+
+  // optional
+  content["annotations"] = nlohmann::json::object();
+  content["annotations"]["audience"] = nlohmann::json::array();
+  content["annotations"]["audience"].push_back("user");
+  content["annotations"]["priority"] = 0.9;
+
+  result["content"].push_back(content);
+  
+  return true;
+}
+
 bool ToUSDA(Context &ctx, const nlohmann::json &args, nlohmann::json &result, std::string &err) {
   DCOUT("args " << args);
   if (!args.contains("uri")) {
@@ -342,6 +433,60 @@ bool GetToolsList(Context &ctx, nlohmann::json &result) {
 
   }
 
+  {
+    nlohmann::json j;
+    j["name"] = "save_screenshot";
+    j["description"] = "Save screenshot image(`data` is a base64 encoded string of image data)";
+
+    nlohmann::json schema;
+    schema["type"] = "object";
+    schema["properties"] = nlohmann::json::object();
+    schema["properties"]["data"] ={{"type", "string"}};
+    schema["properties"]["name"] ={{"type", "string"}};
+    schema["properties"]["mimeType"] ={{"type", "string"}};
+
+    schema["required"] = nlohmann::json::array({"data", "name", "mimeType"});
+
+    j["inputSchema"] = schema;
+
+    result["tools"].push_back(j);
+
+  }
+
+  {
+    nlohmann::json j;
+    j["name"] = "list_screenshots";
+    j["description"] = "List screenshot image names";
+
+    nlohmann::json schema;
+    schema["type"] = "object";
+    schema["properties"] = nlohmann::json::object();
+
+    j["inputSchema"] = schema;
+
+    result["tools"].push_back(j);
+
+  }
+
+  {
+    nlohmann::json j;
+    j["name"] = "read_screenshot";
+    j["description"] = "Read screenshot image";
+
+    nlohmann::json schema;
+    schema["type"] = "object";
+    schema["properties"] = nlohmann::json::object();
+    schema["properties"]["name"] ={{"type", "string"}};
+
+    schema["required"] = nlohmann::json::array({"name"});
+
+    j["inputSchema"] = schema;
+
+    result["tools"].push_back(j);
+
+  }
+
+
   std::cout << result << "\n";
 
   return true;
@@ -365,6 +510,12 @@ bool CallTool(Context &ctx, const std::string &tool_name, const nlohmann::json &
   } else if (tool_name == "list_primspecs") {
     DCOUT("list_primspecs");
     return ListPrimSpecs(ctx, args, result, err);
+  } else if (tool_name == "list_screenshots") {
+    return ListScreenshots(ctx, args, result, err);
+  } else if (tool_name == "save_screenshot") {
+    return SaveScreenshot(ctx, args, result, err);
+  } else if (tool_name == "read_screenshot") {
+    return ReadScreenshot(ctx, args, result, err);
 #if 0
   } else if (tool_name == "get_texture_asset") {
     return GetTextureAsset(ctx, args, result, err);

web/demo/mcp-client.js

@@ -0,0 +1,71 @@
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
+import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js";
+
+import { createCanvas } from "canvas";
+
+
+//await Bun.write("bun.png", canvas.toBuffer());
+
+
+const url = "http://localhost:8085/mcp"
+
+function genImage() {
+  const canvas = createCanvas(50, 50);
+  const ctx = canvas.getContext('2d');
+
+  ctx.fillStyle = "black";
+  ctx.fillRect(0, 0, canvas.width, canvas.height);
+  ctx.fillStyle = "white";
+  ctx.fillText("bora", 0, 30);
+
+  const dataurl = canvas.toDataURL('image/jpeg', /* quality */0.8 );
+  // strip mime prefix
+  return dataurl.replace(/^.*,/, '');
+}
+
+console.log(genImage());
+
+async function sendScreenshot(client) {
+
+    const dataURI = genImage();
+
+    await client.callTool({
+      name: "save_screenshot",
+      arguments: {
+        "uri" : dataURI
+      }
+    });
+}
+
+let client = null;
+const baseUrl = new URL(url);
+try {
+  client = new Client({
+    name: 'streamable-http-client',
+    version: '1.0.0'
+  });
+  const transport = new StreamableHTTPClientTransport(
+    new URL(baseUrl)
+  );
+  await client.connect(transport);
+  console.log("Connected using Streamable HTTP transport");
+
+} catch (error) {
+  // If that fails with a 4xx error, try the older SSE transport
+  console.log("Streamable HTTP connection failed, falling back to SSE transport");
+  client = new Client({
+    name: 'sse-client',
+    version: '1.0.0'
+  });
+  const sseTransport = new SSEClientTransport(baseUrl);
+  await client.connect(sseTransport);
+  console.log("Connected using SSE transport");
+}
+
+const tools = await client.listTools();
+console.log(tools);
+
+//sendScreenshot(client)
+
+

web/demo/package.json

@@ -16,6 +16,7 @@
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.15.1",
+    "canvas": "^3.0.0-rc3",
     "fzstd": "^0.1.1",
     "gsap": "^3.13.0",
     "lil-gui": "^0.19.2",