various post-sql-merge cleanups

* Remove pinned connector images and switch back to :dev tags.

* Update some comments, and regenerate protobuf types.

* Remove labels.PortConfig in favor of the canonical pf.NetworkPort.

* Remove re-exports from ops => labels.

* Clean up a shuffle package test, written before SQLite derivations
  were available, but which depends on read-only tranforms.

* Remove extra wait time for airbyte-to-flow, since resolved.

* Clean up after catalog tests, and ensure we keep the tree clean.

* Regenerate flow.schema.json

No behavior changes.

Author: jgraettinger

.github/workflows/main.yml

@@ -65,17 +65,13 @@ jobs:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
 
-      # Build and test all the things.
       - run: make install-tools
       - run: go mod download
       - run: make rust-gnu-test
       - run: make linux-gnu-binaries
 
-      - name: Ensure that generated files are unchanged.
-        run: |
-          git clean --force -d
-          git diff
-          [[ -z "$(git status --porcelain)" ]] || exit 1
+      - name: Ensure that we didn't dirty the tree.
+        run: git clean --force -d && git diff --exit-code || exit 1
 
       - name: Upload Binaries
         uses: actions/upload-artifact@v2
@@ -112,7 +108,6 @@ jobs:
 
       - uses: Swatinem/rust-cache@v2
 
-      # Build and test all the things.
       - run: go mod download
       - run: make rust-musl-test
       - run: make linux-musl-binaries
@@ -155,7 +150,6 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      # We require a minimal Go version of 1.17.
       - uses: actions/setup-go@v2
         with:
           go-version: "1.19"
@@ -212,9 +206,6 @@ jobs:
       - name: Add PKGDIR/bin to PATH
         run: echo "$GITHUB_WORKSPACE/.build/package/bin" >> $GITHUB_PATH
 
-      # Use make's `-o` flag to mark the binary artifacts they depend on as
-      # "very old", which prevents Make from attempting to rebuild them from
-      # scratch. This prevents us from recompiling them again here.
       - name: make catalog-test
         run: |
           make catalog-test SKIP_BUILD=true
@@ -231,7 +222,9 @@ jobs:
         run: |
           make /home/runner/work/flow/flow/.build/package/flow-x86-linux.tar.gz
 
-      # Package and push images.
+      - name: Ensure that we didn't dirty the tree.
+        run: git clean --force -d && git diff --exit-code || exit 1
+
       - run: make docker-image
 
       - run: make docker-push
@@ -250,17 +243,3 @@ jobs:
           tag_name: dev
           generate_release_notes: false
           files: .build/package/flow-x86-linux.tar.gz
-
-      # TODO remove me when merged.
-      - if: ${{ github.ref == 'refs/pull/985/merge' }}
-        name: Upload release assets (dev-sql)
-        uses: softprops/action-gh-release@v1
-        with:
-          body: |
-            This is the latest development build of the sql branch, which gets updated on every successful build of its branch.
-          target_commitish: ${{ github.sha }}
-          draft: false
-          prerelease: true
-          tag_name: dev-sql
-          generate_release_notes: false
-          files: .build/package/flow-x86-linux.tar.gz

Makefile

@@ -337,6 +337,9 @@ endif
 .PHONY: catalog-test
 catalog-test: data-plane-test-setup
 	${PKGDIR}/bin/flowctl-go test --source examples/flow.yaml $(ARGS)
+	# Cleanup generated SQLite DB.
+	# TODO(johnny): Remove with in-process SQLite.
+	rm examples/examples.db*
 
 .PHONY: end-to-end-test
 end-to-end-test: data-plane-test-setup

crates/proto-flow/src/flow.rs

@@ -677,6 +677,8 @@ pub mod materialization_spec {
         #[allow(clippy::derive_partial_eq_without_eq)]
         #[derive(Clone, PartialEq, ::prost::Message)]
         pub struct DeprecatedShuffle {
+            #[prost(string, tag = "1")]
+            pub group_name: ::prost::alloc::string::String,
             #[prost(message, optional, tag = "3")]
             pub partition_selector: ::core::option::Option<
                 ::proto_gazette::broker::LabelSelector,

crates/proto-flow/src/flow.serde.rs

@@ -4350,10 +4350,16 @@ impl serde::Serialize for materialization_spec::binding::DeprecatedShuffle {
     {
         use serde::ser::SerializeStruct;
         let mut len = 0;
+        if !self.group_name.is_empty() {
+            len += 1;
+        }
         if self.partition_selector.is_some() {
             len += 1;
         }
         let mut struct_ser = serializer.serialize_struct("flow.MaterializationSpec.Binding.DeprecatedShuffle", len)?;
+        if !self.group_name.is_empty() {
+            struct_ser.serialize_field("groupName", &self.group_name)?;
+        }
         if let Some(v) = self.partition_selector.as_ref() {
             struct_ser.serialize_field("partitionSelector", v)?;
         }
@@ -4367,12 +4373,15 @@ impl<'de> serde::Deserialize<'de> for materialization_spec::binding::DeprecatedS
         D: serde::Deserializer<'de>,
     {
         const FIELDS: &[&str] = &[
+            "group_name",
+            "groupName",
             "partition_selector",
             "partitionSelector",
         ];
 
         #[allow(clippy::enum_variant_names)]
         enum GeneratedField {
+            GroupName,
             PartitionSelector,
         }
         impl<'de> serde::Deserialize<'de> for GeneratedField {
@@ -4395,6 +4404,7 @@ impl<'de> serde::Deserialize<'de> for materialization_spec::binding::DeprecatedS
                         E: serde::de::Error,
                     {
                         match value {
+                            "groupName" | "group_name" => Ok(GeneratedField::GroupName),
                             "partitionSelector" | "partition_selector" => Ok(GeneratedField::PartitionSelector),
                             _ => Err(serde::de::Error::unknown_field(value, FIELDS)),
                         }
@@ -4415,9 +4425,16 @@ impl<'de> serde::Deserialize<'de> for materialization_spec::binding::DeprecatedS
                 where
                     V: serde::de::MapAccess<'de>,
             {
+                let mut group_name__ = None;
                 let mut partition_selector__ = None;
                 while let Some(k) = map.next_key()? {
                     match k {
+                        GeneratedField::GroupName => {
+                            if group_name__.is_some() {
+                                return Err(serde::de::Error::duplicate_field("groupName"));
+                            }
+                            group_name__ = Some(map.next_value()?);
+                        }
                         GeneratedField::PartitionSelector => {
                             if partition_selector__.is_some() {
                                 return Err(serde::de::Error::duplicate_field("partitionSelector"));
@@ -4427,6 +4444,7 @@ impl<'de> serde::Deserialize<'de> for materialization_spec::binding::DeprecatedS
                     }
                 }
                 Ok(materialization_spec::binding::DeprecatedShuffle {
+                    group_name: group_name__.unwrap_or_default(),
                     partition_selector: partition_selector__,
                 })
             }

crates/proto-flow/src/ops.rs

@@ -22,6 +22,7 @@ pub struct Meta {
     #[prost(string, tag = "1")]
     pub uuid: ::prost::alloc::string::String,
 }
+/// Log is Flow's unified representation of task logs.
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct Log {
@@ -103,6 +104,8 @@ pub mod log {
         }
     }
 }
+/// Stats is Flow's unified representation of task metrics and statistics.
+///
 /// TODO(johnny): We should evolve this into a consolidated message
 /// having a serde serialization corresponding to the ops/stats
 /// collection.

crates/proto-grpc/src/ops.rs

@@ -22,6 +22,7 @@ pub struct Meta {
     #[prost(string, tag = "1")]
     pub uuid: ::prost::alloc::string::String,
 }
+/// Log is Flow's unified representation of task logs.
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct Log {
@@ -103,6 +104,8 @@ pub mod log {
         }
     }
 }
+/// Stats is Flow's unified representation of task metrics and statistics.
+///
 /// TODO(johnny): We should evolve this into a consolidated message
 /// having a serde serialization corresponding to the ops/stats
 /// collection.

examples/citi-bike/csv-rides.flow.yaml

@@ -7,7 +7,7 @@ captures:
   examples/citi-bike/rides-from-s3:
     endpoint:
       connector:
-        image: ghcr.io/estuary/source-s3:8952e37
+        image: ghcr.io/estuary/source-s3:dev
         config: rides.config.yaml
     bindings:
       - resource: