2 # org.onap.vnfrqts/requirements
3 # ============LICENSE_START====================================================
4 # Copyright © 2018 AT&T Intellectual Property. All rights reserved.
6 # Unless otherwise specified, all software contained herein is licensed
7 # under the Apache License, Version 2.0 (the "License");
8 # you may not use this software except in compliance with the License.
9 # You may obtain a copy of the License at
11 # http://www.apache.org/licenses/LICENSE-2.0
13 # Unless required by applicable law or agreed to in writing, software
14 # distributed under the License is distributed on an "AS IS" BASIS,
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
19 # Unless otherwise specified, all documentation contained herein is licensed
20 # under the Creative Commons License, Attribution 4.0 Intl. (the "License");
21 # you may not use this documentation except in compliance with the License.
22 # You may obtain a copy of the License at
24 # https://creativecommons.org/licenses/by/4.0/
26 # Unless required by applicable law or agreed to in writing, documentation
27 # distributed under the License is distributed on an "AS IS" BASIS,
28 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
29 # See the License for the specific language governing permissions and
30 # limitations under the License.
32 # ============LICENSE_END============================================
35 This script should be run before every commit to ensure proper
36 standards are being followed within the project. The script
37 will also automatically fix certain issues when they are encountered, and
38 warn about other issues it cannot automatically resolve.
41 - Requirement missing required attributes
42 - Invalid values for attributes
43 - Invalid section header usage in any file
44 - :keyword: and requirement mismatch
47 - Assigning :id: on new requirements where an ID missing
48 - Adding :introduced: attribute on new requirements
49 - Adding/correcting :updated: attribute on changed requirements
56 from abc import ABC, abstractmethod
57 from collections import OrderedDict, deque
58 from pathlib import Path
59 from typing import Deque, List, Mapping, Callable, Set
63 THIS_DIR = Path(__file__).parent
64 CONF_PATH = THIS_DIR / "docs/conf.py"
67 "https://nexus.onap.org/service/local/repositories/raw/content"
68 "/org.onap.vnfrqts.requirements/master/needs.json"
71 HEADING_LEVELS = ("-", "^", "~", "+", "*", '"')
73 SPACES = re.compile(r"\s+")
74 REQ_DIRECTIVE_PATTERN = re.compile(r"\.\.\s+req::.*")
75 ATTRIBUTE_PATTERN = re.compile(r"^\s+(:\w+:)\s+(.*)$")
76 VERSION_PATTERN = re.compile(r"version\s+=\s+'(.*?)'")
78 VALID_KEYWORDS = ("MUST", "MUST NOT", "SHOULD", "SHOULD NOT", "MAY", "MAY NOT")
88 REQUIRED_ATTRIBUTES = (":keyword:", ":target:", ":id:")
93 "VNF DOCUMENTATION PACKAGE",
94 "PNF DOCUMENTATION PACKAGE",
95 "VNF or PNF DOCUMENTATION PACKAGE",
98 "VNF or PNF PROVIDER",
101 "VNF or PNF CSAR PACKAGE",
104 VALID_VALIDATION_MODES = ("static", "none", "in_service")
107 def check(predicate: bool, msg: str):
109 Raises RuntimeError with given msg if predicate is False
112 raise RuntimeError(msg)
115 def get_version() -> str:
117 Returns the version value from conf.py
119 with open(CONF_PATH) as f:
121 m = VERSION_PATTERN.match(line)
123 version = m.groups()[0]
124 if version not in VALID_VERSIONS:
126 f"ERROR: {version} in conf.py is not defined in "
127 f"VALID_VERSIONS. Update the script to continue"
133 VERSION = get_version()
136 def normalize(text: str):
138 Strips out formatting, line breaks, and repeated spaces to normalize
139 the string for comparison. This ensures minor formatting changes
140 are not tagged as meaningful changes
143 s = s.replace("\n", " ")
144 s = re.sub(r'[`*\'"]', "", s)
145 s = re.sub(r"\s+", " ", s)
149 def warn(path: str, msg: str, req: "RequirementDirective" = None):
153 req_id = req.requirement_id or "UNKNOWN" if req else "UNKNOWN"
154 print(f"WARNING: {path} | {req_id} | {msg}")
157 class RequirementRepository:
159 Pulls needs.json and provides various options to interact with the data.
162 def __init__(self, data=None):
163 self.data = data or requests.get(NEEDS_JSON_URL).json()
166 for version in self.data["versions"].values()
167 for r in version["needs"].values()
171 def current_requirements(self) -> Mapping:
173 Returns the requirements specified by current_version in needs.json.
175 version = self.data["current_version"]
176 return self.data["versions"][version]["needs"]
179 def unique_targets(self) -> Set[str]:
180 return {r["target"] for r in self.current_requirements.values()}
183 def unique_validation_modes(self) -> Set[str]:
184 return {r["validation_mode"] for r in self.current_requirements.values()}
186 def create_id(self) -> str:
188 Generates a requirements ID that has not been used in any version
192 new_id = "R-{:0>5d}".format(random.randint(0, 99999))
193 if new_id in self.all_ids:
194 continue # skip this one and generate another one
195 self.all_ids.add(new_id)
198 def is_new_requirement(self, req: "RequirementDirective") -> bool:
199 return req.requirement_id not in self.current_requirements
201 def has_changed(self, req: "RequirementDirective") -> bool:
203 Returns True if the requirement already exists and the contents has
204 meaningfully changed. Small changes in spaces or formatting are not considered.
206 current_req = self.current_requirements.get(req.requirement_id)
209 return normalize(current_req["description"]) == normalize("".join(req.content))
212 class RequirementDirective:
214 Data structure to hold a .. req:: directive
227 def __init__(self, path: str):
229 self.attributes = OrderedDict()
234 def requirement_id(self) -> str:
235 return self.attributes.get(":id:", "")
237 @requirement_id.setter
238 def requirement_id(self, r_id: str):
239 self._update(":id:", r_id)
242 def keyword(self) -> str:
243 return self.attributes.get(":keyword:", "")
246 def keyword(self, k: str):
247 self._update(":keyword:", k)
250 def target(self) -> str:
251 return self.attributes.get(":target:", "")
254 def target(self, value: str):
255 self._update(":target", value)
258 def introduced(self) -> str:
259 return self.attributes.get(":introduced:", "")
262 def introduced(self, version: str):
263 self._update(":introduced:", version)
266 def updated(self) -> str:
267 return self.attributes.get(":updated:", "")
270 def updated(self, version: str):
271 self._update(":updated:", version)
274 def validation_mode(self) -> str:
275 return self.attributes.get(":validation_mode:", "")
277 @validation_mode.setter
278 def validation_mode(self, value: str):
279 self._update(":validation_mode:", value)
281 def parse(self, lines: Deque[str]):
283 Parses a ..req:: directive and populates the data structre
287 line = lines.popleft()
288 match = ATTRIBUTE_PATTERN.match(line) if parsing_attrs else None
290 self.indent = self.indent or self.calc_indent(line)
291 attr, value = match.groups()
292 self.attributes[attr] = value
294 parsing_attrs = False # passed attributes, capture content
295 if line.strip() and self.calc_indent(line) < self.indent:
296 # past end of the directive so we'll put this line back
297 lines.appendleft(line)
300 self.content.append(line)
302 def format_attributes(self) -> List[str]:
304 Converts a dict back to properly indented lines using ATTRIBUTE_ORDER
306 spaces = " " * self.indent
308 for key in self.ATTRIBUTE_ORDER:
309 value = self.attributes.get(key)
311 attr_lines.append(f"{spaces}{key} {value}\n")
315 def calc_indent(line: str) -> int:
317 Number of leading spaces of the line
319 return len(line) - len(line.lstrip())
322 return "".join(self.format_attributes() + self.content)
324 def _notify(self, field, value):
325 req_id = self.requirement_id or "UNKNOWN"
326 print(f"UPDATE: {self.path} | {req_id} | Setting {field} to {value}")
328 def _update(self, attr, value):
329 self.attributes[attr] = value
330 self._notify(attr, value)
333 class RequirementVisitor:
335 Walks a directory for reStructuredText files and and passes contents to
336 visitors when the content is encountered.
338 Types of visitors supported:
340 - Requirement: Take the path and a RequirementDirective which may be modified
341 If modified, the file will be updated using the modified directive
342 - Post Processor: Take the path and all lines for processing; returning a
343 potentially changed set of lines
348 req_visitors: List[Callable[[str, RequirementDirective], None]],
349 post_processors: List[Callable[[str, List[str]], List[str]]],
351 self.req_visitors = req_visitors or []
352 self.post_processors = post_processors or []
354 def process(self, root_dir: Path):
356 Walks the `root_dir` looking for rst to files to parse
358 for dir_path, sub_dirs, filenames in os.walk(root_dir.as_posix()):
359 for filename in filenames:
360 if filename.lower().endswith(".rst"):
361 self.handle_rst_file(os.path.join(dir_path, filename))
365 """Read file at `path` and return lines as list"""
366 with open(path, "r") as f:
370 def write(path, content):
371 """Write a content to the given path"""
372 with open(path, "w") as f:
376 def handle_rst_file(self, path):
378 Parse the RST file notifying the registered visitors
380 lines = deque(self.read(path))
383 line = lines.popleft()
384 if self.is_req_directive(line):
385 req = RequirementDirective(path)
387 for func in self.req_visitors:
389 # Put the lines back for processing by the line visitor
390 lines.extendleft(reversed(req.format_attributes() + req.content))
391 new_contents.append(line)
392 for processor in self.post_processors:
393 new_contents = processor(path, new_contents) or new_contents
394 self.write(path, new_contents)
397 def is_req_directive(line):
398 """Returns True if the line denotes the start of a needs directive"""
399 return bool(REQ_DIRECTIVE_PATTERN.match(line))
402 class AbstractRequirementVisitor(ABC):
404 def __call__(self, path: str, req: RequirementDirective):
405 raise NotImplementedError()
408 class MetadataFixer(AbstractRequirementVisitor):
410 Updates metadata based on the status of the requirement and contents of
414 def __init__(self, repos: RequirementRepository):
417 def __call__(self, path: str, req: RequirementDirective):
418 if not req.requirement_id:
419 req.requirement_id = self.repos.create_id()
420 if self.repos.is_new_requirement(req) and req.introduced != VERSION:
421 req.introduced = VERSION
422 if self.repos.has_changed(req) and req.updated != VERSION:
423 req.updated = VERSION
426 class MetadataValidator(AbstractRequirementVisitor):
427 def __init__(self, repos: RequirementRepository):
430 def __call__(self, path: str, req: RequirementDirective):
431 for attr in REQUIRED_ATTRIBUTES:
432 if attr not in req.attributes:
433 warn(path, f"Missing required attribute {attr}", req)
434 if req.keyword and req.keyword not in VALID_KEYWORDS:
435 warn(path, f"Invalid :keyword: value ({req.keyword})", req)
436 if repository.is_new_requirement(req) and req.introduced != VERSION:
437 warn(path, f":introduced: is not {VERSION} on new requirement", req)
438 if req.introduced and req.introduced not in VALID_VERSIONS:
439 warn(path, f"Invalid :introduced: value ({req.introduced})", req)
440 if req.updated and req.updated not in VALID_VERSIONS:
441 warn(path, f"Invalid :updated: value ({req.updated})", req)
442 if req.target and req.target not in VALID_TARGETS:
443 warn(path, f"Invalid :target: value ({req.target})", req)
444 if req.validation_mode and req.validation_mode not in VALID_VALIDATION_MODES:
445 warn(path, f"Invalid :validation_mode: value ({req.validation_mode})", req)
448 def check_section_headers(path: str, lines: List[str]) -> List[str]:
450 Ensure hierarchy of section headers follows the expected progression as defined
451 by `HEADING_LEVELS`, and that section heading marks match the length of the
454 current_heading_level = 0
455 for i, line in enumerate(lines):
456 if any(line.startswith(char * 3) for char in HEADING_LEVELS):
457 # heading level should go down, stay the same, or be next level
458 expected = HEADING_LEVELS[0 : current_heading_level + 2]
459 if line[0] not in expected:
462 f"Unexpected heading char ({line[0]}) on line {i+1}. "
463 f"Expected one of {' '.join(expected)}",
465 if len(line.strip()) != len(lines[i - 1].strip()):
466 lines[i] = (line[0] * len(lines[i - 1].strip())) + "\n"
468 f"UPDATE: {path} | Matching section mark to title length "
471 current_heading_level = HEADING_LEVELS.index(line[0])
475 def check_keyword_text_alignment(path: str, req: RequirementDirective):
478 keyword = f"**{req.keyword}**"
479 if not any(keyword in line for line in req.content):
480 warn(path, f"Keyword is {req.keyword}, but {keyword} not in requirement", req)
483 if __name__ == "__main__":
484 print("Valid Versions")
485 print("-----------------------")
486 print("\n".join(VALID_VERSIONS))
488 print("Valid Keywords")
489 print("-----------------------")
490 print("\n".join(VALID_KEYWORDS))
492 print("Valid Targets")
493 print("-----------------------")
494 print("\n".join(VALID_TARGETS))
496 print("Valid Validation Modes")
497 print("-----------------------")
498 print("\n".join(VALID_VALIDATION_MODES))
500 print("Check-up Report")
502 repository = RequirementRepository()
503 visitor = RequirementVisitor(
505 MetadataFixer(repository),
506 MetadataValidator(repository),
507 check_keyword_text_alignment,
509 post_processors=[check_section_headers],
511 visitor.process(THIS_DIR / "docs")