feat: implement skill-evaluator loop and benchmark aggregation framework

agent/executor.py

@@ -375,7 +375,7 @@ def _system_prompt(self, tables: Dict) -> str:
 {instruction_text}
 
 ## Instructions
-- Use the available tools to interact with tables. Always call `table_info` first to understand structure.
+- Use the available tools to interact with tables. For table-analysis tasks, call `table_info` first to understand structure.
 - For questions about data, retrieve and process the data with tools before answering.
 - When your operation produces a new table, give it a descriptive `result_name`.
 - Multiple tables can be referenced. Cross-table operations (merge, compare) are supported.

config.py

@@ -5,7 +5,14 @@
 def load_settings():
     settings_path = Path(__file__).parent / "setting.txt"
     settings = {}
-    with open(settings_path) as f:
+    # Prefer UTF-8 (with/without BOM) so settings copied from README/docs
+    # work consistently across Windows and Unix environments.
+    try:
+        f = open(settings_path, encoding="utf-8-sig")
+    except UnicodeDecodeError:
+        # Fallback for legacy Windows-local files explicitly saved in GBK.
+        f = open(settings_path, encoding="gbk")
+    with f:
         for line in f:
             line = line.strip()
             if "=" in line and not line.startswith("#"):
@@ -17,7 +24,10 @@ def load_settings():
 _s = load_settings()
 API_KEY = _s.get("API_KEY", "")
 BASE_URL = _s.get("BASE_URL", "https://api.openai.com/v1")
-DEFAULT_MODEL = _s.get("DEFAULT_MODEL", "deepseek-ai/DeepSeek-V3")
+# Backward compatibility:
+# - newer config key: DEFAULT_MODEL
+# - legacy key used in some local setups: MODEL_NAME
+DEFAULT_MODEL = _s.get("DEFAULT_MODEL") or _s.get("MODEL_NAME", "deepseek-ai/DeepSeek-V3")
 DEFAULT_MODEL_EXTRA_JSON = _s.get("DEFAULT_MODEL_EXTRA_JSON", "")
 DEFAULT_MODEL_EXTRA_PARAMS = None
 if DEFAULT_MODEL_EXTRA_JSON != "":

skills/registry.py

@@ -18,6 +18,7 @@
 from skills.workspace_tools import WORKSPACE_TOOL_DEFS, WORKSPACE_SKILLS, WORKSPACE_DIR
 
 SKILLS_DIR = Path(__file__).parent.parent / "data" / "skills"
+BUNDLED_SKILLS_DIR = Path(__file__).parent
 
 
 def _parse_skill_md(text: str) -> Dict:
@@ -41,6 +42,33 @@ def _parse_skill_md(text: str) -> Dict:
 
 # OpenAI-format tool definitions for every built-in skill
 BUILTIN_TOOL_DEFS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "skill-evaluator",
+            "description": (
+                "Create a reusable package skill (SKILL.md) directly from chat. "
+                "Use this when user asks to create a new skill/instruction package."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string", "description": "Human-readable skill name"},
+                    "description": {"type": "string", "description": "One-line summary of what this skill does"},
+                    "body": {
+                        "type": "string",
+                        "description": "Main SKILL.md body content (instructions/rules/examples)",
+                    },
+                    "source": {
+                        "type": "string",
+                        "description": "Optional source tag for metadata",
+                        "default": "manual",
+                    },
+                },
+                "required": ["name", "description", "body"],
+            },
+        },
+    },
     {
         "type": "function",
         "function": {
@@ -344,6 +372,7 @@ def _parse_skill_md(text: str) -> Dict:
 ]
 
 BUILTIN_META = {
+    "skill-evaluator": {"description": "Create a new SKILL.md package from chat", "category": "skill_management"},
     "table_info": {"description": "Get table metadata (shape, columns, dtypes, sample)", "category": "inspection"},
     "filter_rows": {"description": "Filter rows using a query condition", "category": "transformation"},
     "select_columns": {"description": "Select a subset of columns", "category": "transformation"},
@@ -405,45 +434,67 @@ def __init__(self):
     # ------------------------------------------------------------------
 
     def _load_packages(self) -> List[Dict]:
-        """Scan data/skills/*/SKILL.md and load metadata for each."""
-        packages: List[Dict] = []
-        if not SKILLS_DIR.exists():
-            return packages
-        for skill_dir in sorted(SKILLS_DIR.iterdir()):
-            if not skill_dir.is_dir():
-                continue
-            skill_md = skill_dir / "SKILL.md"
-            if not skill_md.exists():
+        """Scan bundled + user skill folders and load all SKILL.md packages.
+
+        Priority: user-installed skills in data/skills override bundled skills
+        with the same slug.
+        """
+        by_slug: Dict[str, Dict[str, Any]] = {}
+        scan_roots = [
+            (BUNDLED_SKILLS_DIR, "bundled"),
+            (SKILLS_DIR, "manual"),
+        ]
+        for root_dir, default_source in scan_roots:
+            if not root_dir.exists():
                 continue
-            parsed = _parse_skill_md(skill_md.read_text(encoding="utf-8"))
-            slug = skill_dir.name
-            meta: Dict[str, Any] = {}
-            meta_path = skill_dir / "_meta.json"
-            if meta_path.exists():
-                try:
-                    meta = json.loads(meta_path.read_text(encoding="utf-8"))
-                except Exception:
-                    pass
-            state_path = skill_dir / "_state.json"
-            state: Dict[str, Any] = {"enabled": True}
-            if state_path.exists():
-                try:
-                    state = json.loads(state_path.read_text(encoding="utf-8"))
-                except Exception:
-                    pass
-            packages.append({
-                "slug": slug,
-                "name": parsed["name"] or slug,
-                "description": parsed["description"],
-                "version": meta.get("version", ""),
-                "source": meta.get("source", "manual"),
-                "enabled": state.get("enabled", True),
-                "body": parsed["body"],
-                "type": "package",
-                "skill_dir": str(skill_dir),
-                "hooks": get_skill_hooks(skill_dir),
-            })
-        return packages
+            for skill_dir in sorted(root_dir.iterdir()):
+                if not skill_dir.is_dir():
+                    continue
+                skill_md = skill_dir / "SKILL.md"
+                if not skill_md.exists():
+                    continue
+                parsed = _parse_skill_md(skill_md.read_text(encoding="utf-8-sig"))
+                slug = skill_dir.name
+                meta: Dict[str, Any] = {}
+                meta_path = skill_dir / "_meta.json"
+                if meta_path.exists():
+                    try:
+                        meta = json.loads(meta_path.read_text(encoding="utf-8"))
+                    except Exception:
+                        pass
+                state_path = skill_dir / "_state.json"
+                state: Dict[str, Any] = {"enabled": True}
+                if state_path.exists():
+                    try:
+                        state = json.loads(state_path.read_text(encoding="utf-8"))
+                    except Exception:
+                        pass
+                by_slug[slug] = {
+                    "slug": slug,
+                    "name": parsed["name"] or slug,
+                    "description": parsed["description"],
+                    "version": meta.get("version", ""),
+                    "source": meta.get("source", default_source),
+                    "enabled": state.get("enabled", True),
+                    "body": parsed["body"],
+                    "type": "package",
+                    "skill_dir": str(skill_dir),
+                    "hooks": get_skill_hooks(skill_dir),
+                }
+        return list(by_slug.values())
+
+    def _get_package_by_slug(self, slug: str) -> Optional[Dict[str, Any]]:
+        return next((p for p in self._packages if p["slug"] == slug), None)
+
+    def _is_bundled_package(self, slug: str) -> bool:
+        pkg = self._get_package_by_slug(slug)
+        return bool(pkg and pkg.get("source") == "bundled")
+
+    def _get_package_dir(self, slug: str) -> Path:
+        pkg = self._get_package_by_slug(slug)
+        if not pkg:
+            raise ValueError(f"Package skill '{slug}' not found")
+        return Path(pkg["skill_dir"])
 
     def create_package(self, name: str, description: str, body: str,
                        source: str = "manual", derived_from: str = "") -> Dict:
@@ -455,7 +506,8 @@ def create_package(self, name: str, description: str, body: str,
 
         base_slug = slug
         counter = 1
-        while (SKILLS_DIR / slug).exists():
+        existing_slugs = {p["slug"] for p in self._packages}
+        while (SKILLS_DIR / slug).exists() or slug in existing_slugs:
             slug = f"{base_slug}-{counter}"
             counter += 1
 
@@ -552,9 +604,9 @@ def install_from_zip(self, zip_bytes: bytes) -> Dict:
 
     def upgrade_package(self, slug: str, new_body: str, reason: str = "") -> Dict:
         """Upgrade an existing skill: back up old version, write new body, bump version."""
-        dest = SKILLS_DIR / slug
-        if not dest.exists():
-            raise ValueError(f"Package skill '{slug}' not found")
+        if self._is_bundled_package(slug):
+            raise ValueError(f"Bundled skill '{slug}' is read-only and cannot be upgraded")
+        dest = self._get_package_dir(slug)
 
         meta_path = dest / "_meta.json"
         meta: Dict[str, Any] = {}
@@ -603,9 +655,9 @@ def upgrade_package(self, slug: str, new_body: str, reason: str = "") -> Dict:
         }
 
     def delete_package(self, slug: str) -> Dict:
-        dest = SKILLS_DIR / slug
-        if not dest.exists():
-            raise ValueError(f"Package skill '{slug}' not found")
+        if self._is_bundled_package(slug):
+            raise ValueError(f"Bundled skill '{slug}' cannot be deleted")
+        dest = self._get_package_dir(slug)
         shutil.rmtree(dest)
         self._packages = [p for p in self._packages if p["slug"] != slug]
         return {"status": "deleted"}
@@ -621,9 +673,7 @@ def clear_packages(self) -> Dict:
         return {"cleared": count}
 
     def toggle_package(self, slug: str, enabled: bool) -> Dict:
-        dest = SKILLS_DIR / slug
-        if not dest.exists():
-            raise ValueError(f"Package skill '{slug}' not found")
+        dest = self._get_package_dir(slug)
         state_path = dest / "_state.json"
         state_path.write_text(json.dumps({"enabled": enabled}), encoding="utf-8")
         for p in self._packages:
@@ -655,7 +705,10 @@ def run_event_hooks(self, event: str, tool_output: str = "") -> str:
 
     def record_usage(self, slug: str):
         """Increment usage_count and update last_used_at for a package skill."""
-        dest = SKILLS_DIR / slug
+        try:
+            dest = self._get_package_dir(slug)
+        except ValueError:
+            return
         meta_path = dest / "_meta.json"
         if not meta_path.exists():
             return
@@ -669,7 +722,10 @@ def record_usage(self, slug: str):
 
     def record_feedback(self, slug: str, feedback: str):
         """Increment success_count or failure_count based on user feedback."""
-        dest = SKILLS_DIR / slug
+        try:
+            dest = self._get_package_dir(slug)
+        except ValueError:
+            return
         meta_path = dest / "_meta.json"
         if not meta_path.exists():
             return
@@ -752,6 +808,18 @@ def execute_sync(self, skill_name: str, params: Dict, tables: Dict) -> Any:
         """Execute a built-in, workspace, or code skill synchronously."""
         if skill_name in WORKSPACE_SKILLS:
             return WORKSPACE_SKILLS[skill_name](params, tables)
+        if skill_name == "skill-evaluator":
+            name = (params.get("name") or "").strip()
+            description = (params.get("description") or "").strip()
+            body = (params.get("body") or "").strip()
+            source = (params.get("source") or "manual").strip() or "manual"
+            if not name or not description or not body:
+                raise ValueError("skill-evaluator requires non-empty `name`, `description`, and `body`")
+            pkg = self.create_package(name=name, description=description, body=body, source=source)
+            return (
+                f"Skill created: name='{pkg.get('name', name)}', slug='{pkg.get('slug', '')}', "
+                f"enabled={pkg.get('enabled', True)}."
+            )
         if skill_name == "execute_python":
             return execute_python(params, tables)
         if skill_name not in BUILTIN_SKILLS: