From 379c7bad19ba49c839a64c4e7c1dbad758b37b1f Mon Sep 17 00:00:00 2001 From: crankynetmaam Date: Tue, 11 Aug 2026 11:33:13 -0500 Subject: [PATCH] feat(translator): use the shiny new rewritten GoBGP Protobufs and upgrade GoBGP --- .gitignore | 2 +- Makefile | 15 ++-- compose.yml | 4 +- compose/local/translator/Dockerfile | 13 ++-- compose/production/translator/Dockerfile | 13 ++-- translator/src/translator/gobgp.py | 94 +++++++++++------------- translator/src/translator/translator.py | 7 +- 7 files changed, 70 insertions(+), 78 deletions(-) diff --git a/.gitignore b/.gitignore index de45d5c7..7d303f9e 100644 --- a/.gitignore +++ b/.gitignore @@ -347,5 +347,5 @@ coverage.coverage coverage.xml # Ignore copied-back autogenerated grpc library -translator/*pb2*.py* +translator/api .idea/ruff.xml diff --git a/Makefile b/Makefile index 317ae903..d11cbc48 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ # It'd be nice to keep these in sync with the defaults of the Dockerfiles PYTHON_IMAGE_VER ?= 3.12 POSTGRES_IMAGE_VER ?= 18 +# Exported so compose.yml's own ${GOBGP_VERSION} substitution sees it too +export GOBGP_VERSION ?= v4.8.0 ACT := act --rm --container-options "--privileged -u root" --container-architecture linux/amd64 --platform ubuntu-latest=catthehacker/ubuntu:act-latest @@ -30,7 +32,7 @@ behave-translator-feature: compose.override.yml ## build: rebuilds all your containers or a single one if CONTAINER is specified .Phony: build build: compose.override.yml - @docker compose build --build-arg PYTHON_IMAGE_VER=$(PYTHON_IMAGE_VER) --build-arg POSTGRES_IMAGE_VER=$(POSTGRES_IMAGE_VER) $(CONTAINER) + @docker compose build --build-arg PYTHON_IMAGE_VER=$(PYTHON_IMAGE_VER) --build-arg POSTGRES_IMAGE_VER=$(POSTGRES_IMAGE_VER) --build-arg GOBGP_VERSION=$(GOBGP_VERSION) $(CONTAINER) @docker compose up -d --no-deps $(CONTAINER) @docker compose restart $(CONTAINER) @@ -67,15 +69,8 @@ compose.override.yml: ## copy-libs: copy the translator autogenerated libraries into the translator directory .Phony: copy-libs copy-libs: - @docker compose cp translator:/app/gobgp_pb2.py translator/ - @docker compose cp translator:/app/gobgp_pb2.pyi translator/ - @docker compose cp translator:/app/gobgp_pb2_grpc.py translator/ - @docker compose cp translator:/app/attribute_pb2.py translator/ - @docker compose cp translator:/app/attribute_pb2.pyi translator/ - @docker compose cp translator:/app/attribute_pb2_grpc.py translator/ - @docker compose cp translator:/app/capability_pb2.py translator/ - @docker compose cp translator:/app/capability_pb2.pyi translator/ - @docker compose cp translator:/app/capability_pb2_grpc.py translator/ + @rm -rf translator/api + @docker compose cp translator:/app/api translator/api ## django-addr: get the IP and ephemeral port assigned to docker:8000 .Phony: django-addr diff --git a/compose.yml b/compose.yml index b6e68d42..e9761246 100644 --- a/compose.yml +++ b/compose.yml @@ -76,7 +76,7 @@ services: replicas: ${REDIS_REPLICAS:-1} gobgp: - image: jauderho/gobgp:v3.37.0 + image: jauderho/gobgp:${GOBGP_VERSION:-v4.8.0} networks: default: {} sysctls: @@ -87,7 +87,7 @@ services: replicas: ${GOBGP_REPLICAS:-1} gobgp-secondary: - image: jauderho/gobgp:v3.37.0 + image: jauderho/gobgp:${GOBGP_VERSION:-v4.8.0} networks: default: {} sysctls: diff --git a/compose/local/translator/Dockerfile b/compose/local/translator/Dockerfile index ad4cb784..1e8c5519 100644 --- a/compose/local/translator/Dockerfile +++ b/compose/local/translator/Dockerfile @@ -28,15 +28,18 @@ COPY ./translator/src ./translator/src/ # Generate protobuf files # TODO: Make this less jank and don't do the git clone crap? -# Protos don't work well in packages so we generate them in /app/ outside the package so that bare -# `import gobgp_pb2` style imports work. -RUN git clone -b v3.33.0 https://github.com/osrg/gobgp.git gobgp \ - && cd gobgp/api \ +# GoBGP v4 organizes its protos under proto/api/ and the protos import each other +# via "api/*.proto", so we compile from the proto/ root. That generates an `api` +# Python package under /app/, which we then import as `from api import gobgp_pb2`. +ARG GOBGP_VERSION=v4.8.0 +RUN git clone -b ${GOBGP_VERSION} https://github.com/osrg/gobgp.git gobgp \ + && cd gobgp/proto \ && python3 -m grpc_tools.protoc -I./ \ --python_out=/app/ \ --pyi_out=/app/ \ --grpc_python_out=/app/ \ - *.proto \ + api/*.proto \ + && touch /app/api/__init__.py \ && cd /app && rm -rf gobgp # Install the project itself (including the protobuff stuff) again mounting in the project files. diff --git a/compose/production/translator/Dockerfile b/compose/production/translator/Dockerfile index df9f6bb8..e093de52 100644 --- a/compose/production/translator/Dockerfile +++ b/compose/production/translator/Dockerfile @@ -29,15 +29,18 @@ COPY ./translator/src ./translator/src/ # Generate protobuf files # TODO: Make this less jank and don't do the git clone crap? -# Protos don't work well in packages so we generate them in /app/ outside the package so that bare -# `import gobgp_pb2` style imports work. -RUN git clone -b v3.33.0 https://github.com/osrg/gobgp.git gobgp \ - && cd gobgp/api \ +# GoBGP v4 organizes its protos under proto/api/ and the protos import each other +# via "api/*.proto", so we compile from the proto/ root. That generates an `api` +# Python package under /app/, which we then import as `from api import gobgp_pb2`. +ARG GOBGP_VERSION=v4.8.0 +RUN git clone -b ${GOBGP_VERSION} https://github.com/osrg/gobgp.git gobgp \ + && cd gobgp/proto \ && python3 -m grpc_tools.protoc -I./ \ --python_out=/app/ \ --pyi_out=/app/ \ --grpc_python_out=/app/ \ - *.proto \ + api/*.proto \ + && touch /app/api/__init__.py \ && cd /app && rm -rf gobgp # Install the project itself (including the protobuff stuff) again mounting in the project files. diff --git a/translator/src/translator/gobgp.py b/translator/src/translator/gobgp.py index 583a7202..03597636 100644 --- a/translator/src/translator/gobgp.py +++ b/translator/src/translator/gobgp.py @@ -2,11 +2,8 @@ import logging -import attribute_pb2 -import gobgp_pb2 -import gobgp_pb2_grpc import grpc -from google.protobuf.any_pb2 import Any +from api import attribute_pb2, common_pb2, gobgp_pb2, gobgp_pb2_grpc, nlri_pb2 from .exceptions import ASNError from .settings import settings @@ -15,6 +12,7 @@ _TIMEOUT_SECONDS = 1000 MAX_SMALL_ASN = 2**16 MAX_SMALL_COMM = 2**16 +IPV4 = 4 IPV6 = 6 logger = logging.getLogger(__name__) @@ -26,13 +24,12 @@ class GoBGP: def __init__(self, url): """Configure the channel used for communication.""" channel = grpc.insecure_channel(url) - self.stub = gobgp_pb2_grpc.GobgpApiStub(channel) + self.stub = gobgp_pb2_grpc.GoBgpServiceStub(channel) @staticmethod - def _get_family_afi(ip_version): - if ip_version == IPV6: - return gobgp_pb2.Family.AFI_IP6 - return gobgp_pb2.Family.AFI_IP + def _family(ip_version): + afi = common_pb2.Family.AFI_IP6 if ip_version == IPV6 else common_pb2.Family.AFI_IP + return common_pb2.Family(afi=afi, safi=common_pb2.Family.SAFI_UNICAST) def _build_path(self, ip, event_data=None): # noqa: PLR0914 # Grab ASN and Community from our event_data, or use the defaults @@ -41,67 +38,56 @@ def _build_path(self, ip, event_data=None): # noqa: PLR0914 asn = event_data.get("asn", settings.default_asn) community = event_data.get("community", settings.default_community) ip_version = ip.ip.version + family = self._family(ip_version) + + # Make sure our asn is an acceptable value. + asn_is_valid(asn) # Set the origin to incomplete (options are IGP, EGP, incomplete) # Incomplete means that BGP is unsure of exactly how the prefix was injected into the topology. # The most common scenario here is that the prefix was redistributed into Border Gateway Protocol # from some other protocol, typically an IGP. - https://www.kwtrain.com/blog/bgp-pt2 - origin = Any() - origin.Pack( - attribute_pb2.OriginAttribute( - origin=2, - ), - ) + origin = attribute_pb2.Attribute(origin=attribute_pb2.OriginAttribute(origin=2)) # IP prefix and its associated length - nlri = Any() - nlri.Pack( - attribute_pb2.IPAddressPrefix( - prefix_len=ip.network.prefixlen, - prefix=str(ip.ip), - ), + nlri = nlri_pb2.NLRI( + prefix=nlri_pb2.IPAddressPrefix(prefix_len=ip.network.prefixlen, prefix=str(ip.ip)), ) # Set the next hop to the correct value depending on IP family - next_hop = Any() - family_afi = self._get_family_afi(ip_version) if ip_version == IPV6: next_hops = event_data.get("next_hop", settings.default_v6_nexthop) - next_hop.Pack( - attribute_pb2.MpReachNLRIAttribute( - family=gobgp_pb2.Family(afi=family_afi, safi=gobgp_pb2.Family.SAFI_UNICAST), + next_hop = attribute_pb2.Attribute( + mp_reach=attribute_pb2.MpReachNLRIAttribute( + family=family, next_hops=[next_hops], nlris=[nlri], ), ) else: next_hops = event_data.get("next_hop", settings.default_v4_nexthop) - next_hop.Pack( - attribute_pb2.NextHopAttribute( + next_hop = attribute_pb2.Attribute( + next_hop=attribute_pb2.NextHopAttribute( next_hop=next_hops, ), ) # Set our AS Path - as_path = Any() - as_segment = None - - # Make sure our asn is an acceptable value. - asn_is_valid(asn) - as_segment = [attribute_pb2.AsSegment(numbers=[asn])] + as_segment = [attribute_pb2.AsSegment(type=attribute_pb2.AsSegment.TYPE_AS_SEQUENCE, numbers=[asn])] as_segments = attribute_pb2.AsPathAttribute(segments=as_segment) - as_path.Pack(as_segments) + as_path = attribute_pb2.Attribute(as_path=as_segments) # Set our community number # The ASN gets packed into the community so we need to be careful about size to not overflow the structure - communities = Any() # Standard community # Since we pack both into the community string we need to make sure they will both fit if asn < MAX_SMALL_ASN and community < MAX_SMALL_COMM: # We bitshift ASN left by 16 so that there is room to add the community on the end of it. This is because # GoBGP wants the community sent as a single integer. comm_id = (asn << 16) + community - communities.Pack(attribute_pb2.CommunitiesAttribute(communities=[comm_id])) + communities = attribute_pb2.Attribute( + communities=attribute_pb2.CommunitiesAttribute(communities=[comm_id]), + ) else: logger.info("LargeCommunity Used - ASN: %s. Community: %s", asn, community) global_admin = asn @@ -113,14 +99,16 @@ def _build_path(self, ip, event_data=None): # noqa: PLR0914 local_data1=local_data1, local_data2=local_data2, ) - communities.Pack(attribute_pb2.LargeCommunitiesAttribute(communities=[large_community])) + communities = attribute_pb2.Attribute( + large_communities=attribute_pb2.LargeCommunitiesAttribute(communities=[large_community]), + ) attributes = [origin, next_hop, as_path, communities] return gobgp_pb2.Path( nlri=nlri, pattrs=attributes, - family=gobgp_pb2.Family(afi=family_afi, safi=gobgp_pb2.Family.SAFI_UNICAST), + family=family, ) def add_path(self, ip, event_data): @@ -130,7 +118,7 @@ def add_path(self, ip, event_data): path = self._build_path(ip, event_data) self.stub.AddPath( - gobgp_pb2.AddPathRequest(table_type=gobgp_pb2.GLOBAL, path=path), + gobgp_pb2.AddPathRequest(table_type=gobgp_pb2.TABLE_TYPE_GLOBAL, path=path), _TIMEOUT_SECONDS, ) except ASNError as e: @@ -140,7 +128,12 @@ def del_all_paths(self): """Remove all routes from being announced.""" logger.warning("Withdrawing ALL routes") - self.stub.DeletePath(gobgp_pb2.DeletePathRequest(table_type=gobgp_pb2.GLOBAL), _TIMEOUT_SECONDS) + # GoBGP v4 needs an address family set to be able to delete all prefixes for that family. + for ip_version in (IPV4, IPV6): + self.stub.DeletePath( + gobgp_pb2.DeletePathRequest(table_type=gobgp_pb2.TABLE_TYPE_GLOBAL, family=self._family(ip_version)), + _TIMEOUT_SECONDS, + ) def del_path(self, ip, event_data): """Remove a single route from being announced.""" @@ -148,7 +141,7 @@ def del_path(self, ip, event_data): try: path = self._build_path(ip, event_data) self.stub.DeletePath( - gobgp_pb2.DeletePathRequest(table_type=gobgp_pb2.GLOBAL, path=path), + gobgp_pb2.DeletePathRequest(table_type=gobgp_pb2.TABLE_TYPE_GLOBAL, path=path), _TIMEOUT_SECONDS, ) except ASNError as e: @@ -161,33 +154,32 @@ def get_prefixes(self, ip): list: The routes that overlap with the prefix and are currently announced. """ prefixes = [gobgp_pb2.TableLookupPrefix(prefix=str(ip.ip))] - family_afi = self._get_family_afi(ip.ip.version) result = self.stub.ListPath( gobgp_pb2.ListPathRequest( - table_type=gobgp_pb2.GLOBAL, + table_type=gobgp_pb2.TABLE_TYPE_GLOBAL, prefixes=prefixes, - family=gobgp_pb2.Family(afi=family_afi, safi=gobgp_pb2.Family.SAFI_UNICAST), + family=self._family(ip.ip.version), ), _TIMEOUT_SECONDS, ) return list(result) - def get_route_count(self, family_afi): - """Return the number of routes in the global RIB for a given AFI.""" + def get_route_count(self, ip_version): + """Return the number of routes in the global RIB for a given IP version.""" try: result = list( self.stub.ListPath( gobgp_pb2.ListPathRequest( - table_type=gobgp_pb2.GLOBAL, - family=gobgp_pb2.Family(afi=family_afi, safi=gobgp_pb2.Family.SAFI_UNICAST), + table_type=gobgp_pb2.TABLE_TYPE_GLOBAL, + family=self._family(ip_version), ), _TIMEOUT_SECONDS, ) ) - logger.info("GoBGP returned %d routes for family %s", len(result), family_afi) + logger.info("GoBGP returned %d routes for IPv%s", len(result), ip_version) return len(result) except Exception: - logger.exception("Failed to get route count for AFI %s", family_afi) + logger.exception("Failed to get route count for IPv%s", ip_version) return 0 def is_blocked(self, ip): diff --git a/translator/src/translator/translator.py b/translator/src/translator/translator.py index a01de10d..38011d4d 100644 --- a/translator/src/translator/translator.py +++ b/translator/src/translator/translator.py @@ -7,11 +7,10 @@ import json import logging -import gobgp_pb2 import websockets from grpc import RpcError -from .gobgp import GoBGP +from .gobgp import IPV4, IPV6, GoBGP from .settings import DebuggerTypes, settings logging.basicConfig(level=settings.log_level) @@ -79,8 +78,8 @@ async def heartbeat(websocket, g): """Periodically send health status/route counts to Django.""" while True: try: - v4_count = g.get_route_count(gobgp_pb2.Family.AFI_IP) - v6_count = g.get_route_count(gobgp_pb2.Family.AFI_IP6) + v4_count = g.get_route_count(IPV4) + v6_count = g.get_route_count(IPV6) payload = { "type": "translator_heartbeat", "message": {