Run & fix linters flake8, pylint, mypy (#62)

.github/workflows/semconvgen.yml

@@ -12,19 +12,30 @@ on:
 jobs:
  tests:
  runs-on: ubuntu-latest
+ defaults:
+ run: 
+ working-directory: semantic-conventions/
  steps:
  - uses: actions/checkout@v1
  - uses: actions/setup-python@v2
- - name: install dependencies
+ - name: install dev-dependencies
  run: |
- pip install -U pip setuptools wheel
- pip install -r semantic-conventions/dev-requirements.txt
- - name: Check formatting
- run: black --check --diff semantic-conventions/
+ pip install -U pip
+ pip install -U setuptools wheel
+ pip install -r dev-requirements.txt
+ - name: Check formatting (black)
+ run: black --check --diff .
+ - name: Fast basic linting (flake8)
+ run: flake8 .
  - name: install
- run: pip install -U -e ./semantic-conventions/
+ run: pip install -U -e .
  - name: run tests
  run: pytest -v
+ - name: Slow linting (pylint)
+ run: pylint *.py src/
+ - name: Type checking (mypy)
+ run: mypy src/
+
 
  build-and-publish-docker:
  runs-on: ubuntu-latest

semantic-conventions/.flake8

@@ -0,0 +1,15 @@
+# Adapted from https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#flake8
+[flake8]
+extend-ignore = E203
+max-line-length = 120
+exclude =
+ .bzr
+ .git
+ .hg
+ .svn
+ .tox
+ CVS
+ .venv*/
+ venv*/
+ target
+ __pycache__

semantic-conventions/.pylintrc

@@ -0,0 +1,7 @@
+[MESSAGES CONTROL]
+# TODO: We should actually fix the warnings instead of disabling them
+# except for the stuff up to fixme, which is fine to ignore / defer to black. 
+disable = bad-whitespace, wrong-import-order, wrong-hanging-indentation, line-too-long, missing-docstring, unused-import, fixme, invalid-name, too-many-instance-attributes, too-many-locals, too-many-statements, too-many-branches, too-many-arguments, too-many-public-methods, too-few-public-methods, protected-access
+
+[FORMAT]
+good-names=e,ex

semantic-conventions/dev-requirements.txt

@@ -1,3 +1,5 @@
-black==21.6b0
-mypy==0.770
-pytest==6.1.1
+black==21.8b0
+mypy==0.910
+pytest==6.2.5
+flake8==3.9.2
+pylint==2.10.2

semantic-conventions/mypy.ini

@@ -0,0 +1,7 @@
+[mypy]
+
+[mypy-jinja2.*]
+ignore_missing_imports = True
+
+[mypy-mistune.*]
+ignore_missing_imports = True

semantic-conventions/setup.py

@@ -8,7 +8,7 @@
 )
 PACKAGE_INFO = {}
 with open(VERSION_FILENAME, encoding="utf-8") as f:
- exec(f.read(), PACKAGE_INFO)
+ exec(f.read(), PACKAGE_INFO) # pylint:disable=exec-used
 
 
 VERSION_SUFFIX = os.environ.get("SEMCONGEN_VERSION_SUFFIX")

semantic-conventions/src/opentelemetry/semconv/main.py

@@ -20,12 +20,8 @@
 from typing import List
 
 from opentelemetry.semconv.model.semantic_convention import (
+ CONVENTION_CLS_BY_GROUP_TYPE,
  SemanticConventionSet,
- SpanSemanticConvention,
- ResourceSemanticConvention,
- EventSemanticConvention,
- MetricSemanticConvention,
- UnitSemanticConvention,
 )
 from opentelemetry.semconv.templating.code import CodeRenderer
 
@@ -206,13 +202,7 @@ def setup_parser():
  )
  parser.add_argument(
  "--only",
- choices=[
- SpanSemanticConvention.GROUP_TYPE_NAME,
- ResourceSemanticConvention.GROUP_TYPE_NAME,
- EventSemanticConvention.GROUP_TYPE_NAME,
- MetricSemanticConvention.GROUP_TYPE_NAME,
- UnitSemanticConvention.GROUP_TYPE_NAME,
- ],
+ choices=list(CONVENTION_CLS_BY_GROUP_TYPE.keys()),
  help="Process only semantic conventions of the specified type.",
  )
  parser.add_argument(

semantic-conventions/src/opentelemetry/semconv/model/constraints.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from dataclasses import dataclass, field, replace
-from typing import List, Tuple, Set
+from dataclasses import dataclass, replace
+from typing import List, Tuple
 
 from opentelemetry.semconv.model.exceptions import ValidationError
 from opentelemetry.semconv.model.semantic_attribute import SemanticAttribute
@@ -39,9 +39,9 @@ class AnyOf:
  _yaml_src_position Contains the position in the YAML file of the AnyOf attribute
  """
 
- choice_list_ids: Tuple[Tuple[str, ...]]
+ choice_list_ids: Tuple[Tuple[str, ...], ...]
  inherited: bool = False
- choice_list_attributes: Tuple[Tuple[SemanticAttribute, ...]] = ()
+ choice_list_attributes: Tuple[Tuple[SemanticAttribute, ...], ...] = ()
  _yaml_src_position: int = 0
 
  def __eq__(self, other):

semantic-conventions/src/opentelemetry/semconv/model/exceptions.py

@@ -28,7 +28,7 @@ def from_yaml_pos(cls, pos, msg):
  return cls(pos[0] + 1, pos[1] + 1, msg)
 
  def __init__(self, line, column, message):
- super(ValidationError, self).__init__(line, column, message)
+ super().__init__(line, column, message)
  self.message = message
  self.line = line
  self.column = column

semantic-conventions/src/opentelemetry/semconv/model/semantic_attribute.py

@@ -12,12 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import numbers
 import re
 from collections.abc import Iterable
 from dataclasses import dataclass, replace
 from enum import Enum
-from typing import List, Union
+from typing import List, Optional, Union, Dict
 
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
 
@@ -41,20 +40,6 @@ class StabilityLevel(Enum):
  DEPRECATED = 3
 
 
-class HasAttributes:
- def _set_attributes(self, prefix, stability, node):
- self.attrs_by_name = SemanticAttribute.parse(
- prefix, stability, node.get("attributes")
- )
-
- @property
- def attributes(self):
- if not hasattr(self, "attrs_by_name"):
- return []
-
- return list(self.attrs_by_name.values())
-
-
 def unique_attributes(attributes):
  output = []
  for x in attributes:
@@ -97,11 +82,13 @@ def is_enum(self):
  return isinstance(self.attr_type, EnumAttributeType)
 
  @staticmethod
- def parse(prefix, semconv_stability, yaml_attributes):
+ def parse(
+ prefix, semconv_stability, yaml_attributes
+ ) -> "Dict[str, SemanticAttribute]":
  """This method parses the yaml representation for semantic attributes
  creating the respective SemanticAttribute objects.
  """
- attributes = {}
+ attributes = {} # type: Dict[str, SemanticAttribute]
  allowed_keys = (
  "id",
  "type",
@@ -149,6 +136,7 @@ def parse(prefix, semconv_stability, yaml_attributes):
  }
  required_msg = ""
  required_val = attribute.get("required", "")
+ required: Optional[Required]
  if isinstance(required_val, CommentedMap):
  required = Required.CONDITIONAL
  required_msg = required_val.get("conditional", None)
@@ -217,7 +205,7 @@ def parse(prefix, semconv_stability, yaml_attributes):
  "Attribute id "
  + fqn
  + " is already present at line "
- + str(attributes.get(fqn).position[0] + 1)
+ + str(attributes[fqn].position[0] + 1)
  )
  raise ValidationError.from_yaml_pos(position, msg)
  attributes[fqn] = attr
@@ -397,7 +385,7 @@ def to_bool(key, parent_object):
  if isinstance(yaml_value, str):
  if AttributeType.bool_type_true.fullmatch(yaml_value):
  return True
- elif AttributeType.bool_type_false.fullmatch(yaml_value):
+ if AttributeType.bool_type_false.fullmatch(yaml_value):
  return False
  position = parent_object.lc.data[key]
  msg = "Value '{}' for {} field is not allowed".format(yaml_value, key)
@@ -431,47 +419,46 @@ def parse(attribute_type):
  if isinstance(attribute_type, str):
  if AttributeType.is_simple_type(attribute_type):
  return attribute_type
- else: # Wrong type used - rise the exception and fill the missing data in the parent
+ # Wrong type used - raise the exception and fill the missing data in the parent
+ raise ValidationError(
+ 0, 0, "Invalid type: {} is not allowed".format(attribute_type)
+ )
+ allowed_keys = ["allow_custom_values", "members"]
+ mandatory_keys = ["members"]
+ validate_values(attribute_type, allowed_keys, mandatory_keys)
+ custom_values = (
+ bool(attribute_type.get("allow_custom_values"))
+ if "allow_custom_values" in attribute_type
+ else False
+ )
+ members = []
+ if attribute_type["members"] is None or len(attribute_type["members"]) < 1:
+ # Missing members - rise the exception and fill the missing data in the parent
+ raise ValidationError(0, 0, "Enumeration without values!")
+
+ allowed_keys = ["id", "value", "brief", "note"]
+ mandatory_keys = ["id", "value"]
+ for member in attribute_type["members"]:
+ validate_values(member, allowed_keys, mandatory_keys)
+ if not EnumAttributeType.is_valid_enum_value(member["value"]):
  raise ValidationError(
- 0, 0, "Invalid type: {} is not allowed".format(attribute_type)
+ 0, 0, "Invalid value used in enum: <{}>".format(member["value"])
  )
- else:
- allowed_keys = ["allow_custom_values", "members"]
- mandatory_keys = ["members"]
- validate_values(attribute_type, allowed_keys, mandatory_keys)
- custom_values = (
- bool(attribute_type.get("allow_custom_values"))
- if "allow_custom_values" in attribute_type
- else False
- )
- members = []
- if attribute_type["members"] is None or len(attribute_type["members"]) < 1:
- # Missing members - rise the exception and fill the missing data in the parent
- raise ValidationError(0, 0, "Enumeration without values!")
-
- allowed_keys = ["id", "value", "brief", "note"]
- mandatory_keys = ["id", "value"]
- for member in attribute_type["members"]:
- validate_values(member, allowed_keys, mandatory_keys)
- if not EnumAttributeType.is_valid_enum_value(member["value"]):
- raise ValidationError(
- 0, 0, "Invalid value used in enum: <{}>".format(member["value"])
- )
- members.append(
- EnumMember(
- member_id=member["id"],
- value=member["value"],
- brief=member.get("brief").strip()
- if "brief" in member
- else member["id"],
- note=member.get("note").strip() if "note" in member else "",
- )
+ members.append(
+ EnumMember(
+ member_id=member["id"],
+ value=member["value"],
+ brief=member.get("brief").strip()
+ if "brief" in member
+ else member["id"],
+ note=member.get("note").strip() if "note" in member else "",
  )
- enum_type = AttributeType.get_type(members[0].value)
- for m in members:
- if enum_type != AttributeType.get_type(m.value):
- raise ValidationError(0, 0, "Enumeration type inconsistent!")
- return EnumAttributeType(custom_values, members, enum_type)
+ )
+ enum_type = AttributeType.get_type(members[0].value)
+ for m in members:
+ if enum_type != AttributeType.get_type(m.value):
+ raise ValidationError(0, 0, "Enumeration type inconsistent!")
+ return EnumAttributeType(custom_values, members, enum_type)
 
 
 @dataclass