Coverage for src/ollamapy/skill_editor/validator.py: 80%

1"""Skill validation utilities.""" 

2 

3import ast 

4import re 

5from typing import List, Dict, Any 

6from dataclasses import dataclass 

7 

8 

9@dataclass 

10class ValidationResult: 

11 """Result of skill validation.""" 

12 

13 is_valid: bool 

14 errors: List[str] 

15 warnings: List[str] 

16 

17 

18class SkillValidator: 

19 """Validates skill data and code.""" 

20 

21 def __init__(self): 

22 """Initialize the validator.""" 

23 self.required_fields = ["name", "description", "function_code"] 

24 self.valid_parameter_types = ["string", "number", "boolean"] 

25 self.valid_roles = [ 

26 "general", 

27 "text_processing", 

28 "mathematics", 

29 "data_analysis", 

30 "file_operations", 

31 "web_utilities", 

32 "time_date", 

33 "formatting", 

34 "validation", 

35 "emotional_response", 

36 "information", 

37 "advanced", 

38 ] 

39 

40 def validate_skill_data(self, skill_data: Dict[str, Any]) -> ValidationResult: 

41 """Validate complete skill data structure. 

42 

43 Args: 

44 skill_data: The skill data dictionary to validate 

45 

46 Returns: 

47 ValidationResult with validation status and messages 

48 """ 

49 errors = [] 

50 warnings = [] 

51 

52 # Check required fields 

53 for field in self.required_fields: 

54 if field not in skill_data or not skill_data[field]: 

55 errors.append(f"Missing required field: {field}") 

56 

57 # Validate skill name 

58 if "name" in skill_data: 

59 name_validation = self._validate_skill_name(skill_data["name"]) 

60 errors.extend(name_validation.errors) 

61 warnings.extend(name_validation.warnings) 

62 

63 # Validate description 

64 if "description" in skill_data: 

65 desc_validation = self._validate_description(skill_data["description"]) 

66 warnings.extend(desc_validation.warnings) 

67 

68 # Validate role 

69 if "role" in skill_data: 

70 role_validation = self._validate_role(skill_data["role"]) 

71 errors.extend(role_validation.errors) 

72 warnings.extend(role_validation.warnings) 

73 

74 # Validate parameters 

75 if "parameters" in skill_data and skill_data["parameters"]: 

76 param_validation = self._validate_parameters(skill_data["parameters"]) 

77 errors.extend(param_validation.errors) 

78 warnings.extend(param_validation.warnings) 

79 

80 # Validate vibe test phrases 

81 if "vibe_test_phrases" in skill_data: 

82 vibe_validation = self._validate_vibe_phrases( 

83 skill_data["vibe_test_phrases"] 

84 ) 

85 warnings.extend(vibe_validation.warnings) 

86 

87 # Validate function code 

88 if "function_code" in skill_data: 

89 code_validation = self._validate_function_code( 

90 skill_data["function_code"], skill_data.get("parameters", {}) 

91 ) 

92 errors.extend(code_validation.errors) 

93 warnings.extend(code_validation.warnings) 

94 

95 return ValidationResult( 

96 is_valid=len(errors) == 0, errors=errors, warnings=warnings 

97 ) 

98 

99 def _validate_skill_name(self, name: str) -> ValidationResult: 

100 """Validate skill name.""" 

101 errors = [] 

102 warnings = [] 

103 

104 if not isinstance(name, str): 

105 errors.append("Skill name must be a string") 

106 return ValidationResult(False, errors, warnings) 

107 

108 if not name.strip(): 

109 errors.append("Skill name cannot be empty") 

110 return ValidationResult(False, errors, warnings) 

111 

112 # Check for valid identifier 

113 if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", name): 

114 errors.append( 

115 "Skill name must be a valid Python identifier (no spaces or special characters except underscore)" 

116 ) 

117 

118 # Length check 

119 if len(name) > 50: 

120 warnings.append("Skill name is quite long, consider shortening it") 

121 

122 # Naming conventions 

123 if name[0].isupper(): 

124 warnings.append( 

125 "Skill names typically use camelCase or snake_case (starting with lowercase)" 

126 ) 

127 

128 return ValidationResult(len(errors) == 0, errors, warnings) 

129 

130 def _validate_description(self, description: str) -> ValidationResult: 

131 """Validate skill description.""" 

132 errors = [] 

133 warnings = [] 

134 

135 if not isinstance(description, str): 

136 errors.append("Description must be a string") 

137 return ValidationResult(False, errors, warnings) 

138 

139 if not description.strip(): 

140 warnings.append( 

141 "Description is empty - consider adding a clear description of when to use this skill" 

142 ) 

143 return ValidationResult(True, errors, warnings) 

144 

145 if len(description) < 10: 

146 warnings.append( 

147 "Description is very short - consider providing more detail" 

148 ) 

149 

150 if len(description) > 500: 

151 warnings.append( 

152 "Description is very long - consider making it more concise" 

153 ) 

154 

155 # Check for common patterns 

156 if not any( 

157 word in description.lower() for word in ["when", "use", "for", "to"] 

158 ): 

159 warnings.append( 

160 "Description should clearly indicate when this skill should be used" 

161 ) 

162 

163 return ValidationResult(True, errors, warnings) 

164 

165 def _validate_role(self, role: str) -> ValidationResult: 

166 """Validate skill role.""" 

167 errors = [] 

168 warnings = [] 

169 

170 if not isinstance(role, str): 

171 errors.append("Role must be a string") 

172 return ValidationResult(False, errors, warnings) 

173 

174 if role not in self.valid_roles: 

175 errors.append( 

176 f"Invalid role '{role}'. Valid roles: {', '.join(self.valid_roles)}" 

177 ) 

178 

179 return ValidationResult(len(errors) == 0, errors, warnings) 

180 

181 def _validate_parameters(self, parameters: Dict[str, Any]) -> ValidationResult: 

182 """Validate skill parameters.""" 

183 errors = [] 

184 warnings = [] 

185 

186 if not isinstance(parameters, dict): 

187 errors.append("Parameters must be a dictionary") 

188 return ValidationResult(False, errors, warnings) 

189 

190 for param_name, param_info in parameters.items(): 

191 # Validate parameter name 

192 if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", param_name): 

193 errors.append( 

194 f"Parameter name '{param_name}' must be a valid Python identifier" 

195 ) 

196 

197 # Validate parameter info structure 

198 if not isinstance(param_info, dict): 

199 errors.append(f"Parameter '{param_name}' info must be a dictionary") 

200 continue 

201 

202 # Check required fields 

203 if "type" not in param_info: 

204 errors.append(f"Parameter '{param_name}' is missing 'type' field") 

205 elif param_info["type"] not in self.valid_parameter_types: 

206 errors.append( 

207 f"Parameter '{param_name}' has invalid type '{param_info['type']}'. Valid types: {', '.join(self.valid_parameter_types)}" 

208 ) 

209 

210 # Check for description 

211 if "description" not in param_info or not param_info["description"]: 

212 warnings.append(f"Parameter '{param_name}' is missing a description") 

213 

214 # Check required field 

215 if "required" in param_info and not isinstance( 

216 param_info["required"], bool 

217 ): 

218 errors.append( 

219 f"Parameter '{param_name}' 'required' field must be boolean" 

220 ) 

221 

222 return ValidationResult(len(errors) == 0, errors, warnings) 

223 

224 def _validate_vibe_phrases(self, vibe_phrases: List[str]) -> ValidationResult: 

225 """Validate vibe test phrases.""" 

226 errors = [] 

227 warnings = [] 

228 

229 if not isinstance(vibe_phrases, list): 

230 errors.append("Vibe test phrases must be a list") 

231 return ValidationResult(False, errors, warnings) 

232 

233 if not vibe_phrases: 

234 warnings.append( 

235 "No vibe test phrases provided - the AI won't know when to use this skill" 

236 ) 

237 return ValidationResult(True, errors, warnings) 

238 

239 if len(vibe_phrases) < 2: 

240 warnings.append( 

241 "Consider adding more vibe test phrases for better AI recognition" 

242 ) 

243 

244 for i, phrase in enumerate(vibe_phrases): 

245 if not isinstance(phrase, str): 

246 errors.append(f"Vibe test phrase {i+1} must be a string") 

247 elif not phrase.strip(): 

248 warnings.append(f"Vibe test phrase {i+1} is empty") 

249 elif len(phrase) < 5: 

250 warnings.append(f"Vibe test phrase {i+1} is very short") 

251 

252 return ValidationResult(len(errors) == 0, errors, warnings) 

253 

254 def _validate_function_code( 

255 self, code: str, parameters: Dict[str, Any] = None 

256 ) -> ValidationResult: 

257 """Validate Python function code.""" 

258 errors = [] 

259 warnings = [] 

260 

261 if not isinstance(code, str): 

262 errors.append("Function code must be a string") 

263 return ValidationResult(False, errors, warnings) 

264 

265 if not code.strip(): 

266 errors.append("Function code cannot be empty") 

267 return ValidationResult(False, errors, warnings) 

268 

269 # Try to parse the code 

270 try: 

271 tree = ast.parse(code) 

272 except SyntaxError as e: 

273 errors.append(f"Syntax error in function code: {e}") 

274 return ValidationResult(False, errors, warnings) 

275 

276 # Check for execute function 

277 has_execute = False 

278 execute_func = None 

279 

280 for node in ast.walk(tree): 

281 if isinstance(node, ast.FunctionDef) and node.name == "execute": 

282 has_execute = True 

283 execute_func = node 

284 break 

285 

286 if not has_execute: 

287 errors.append("Function code must define an 'execute' function") 

288 return ValidationResult(False, errors, warnings) 

289 

290 # Validate execute function signature 

291 if execute_func and parameters: 

292 param_validation = self._validate_execute_signature( 

293 execute_func, parameters 

294 ) 

295 errors.extend(param_validation.errors) 

296 warnings.extend(param_validation.warnings) 

297 

298 # Check for log usage 

299 if "log(" not in code: 

300 warnings.append( 

301 "Function should use log() to output results that the AI can see" 

302 ) 

303 

304 # Check for potentially dangerous operations 

305 dangerous_patterns = [ 

306 ("os.system", "Using os.system() can be dangerous"), 

307 ("subprocess.call", "Using subprocess.call() can be dangerous"), 

308 ("eval(", "Using eval() can be dangerous"), 

309 ("exec(", "Using exec() can be dangerous"), 

310 ("__import__", "Dynamic imports can be dangerous"), 

311 ] 

312 

313 for pattern, warning in dangerous_patterns: 

314 if pattern in code: 

315 warnings.append(warning) 

316 

317 return ValidationResult(len(errors) == 0, errors, warnings) 

318 

319 def _validate_execute_signature( 

320 self, func_node: ast.FunctionDef, parameters: Dict[str, Any] 

321 ) -> ValidationResult: 

322 """Validate that execute function signature matches declared parameters.""" 

323 errors = [] 

324 warnings = [] 

325 

326 # Get function arguments 

327 func_args = [arg.arg for arg in func_node.args.args] 

328 

329 # Check if all declared parameters are in function signature 

330 for param_name in parameters.keys(): 

331 if param_name not in func_args: 

332 if parameters[param_name].get("required", False): 

333 errors.append( 

334 f"Required parameter '{param_name}' not found in execute function signature" 

335 ) 

336 else: 

337 warnings.append( 

338 f"Optional parameter '{param_name}' not found in execute function signature" 

339 ) 

340 

341 # Check for extra function arguments 

342 param_names = set(parameters.keys()) 

343 for arg_name in func_args: 

344 if arg_name not in param_names: 

345 warnings.append( 

346 f"Function argument '{arg_name}' not declared in parameters" 

347 ) 

348 

349 return ValidationResult(len(errors) == 0, errors, warnings) 

350 

351 def validate_code_syntax(self, code: str) -> ValidationResult: 

352 """Quick syntax validation for code.""" 

353 errors = [] 

354 warnings = [] 

355 

356 try: 

357 ast.parse(code) 

358 except SyntaxError as e: 

359 errors.append(f"Syntax error: {e}") 

360 

361 return ValidationResult(len(errors) == 0, errors, warnings)