Create changelog for datafusion and ballista release (apache#801)

* create changelog for datafusion and ballista release

Created changelog for the following projects:

* datafusion 5.0.0
* python 0.3.0
* ballista 0.5.0

Other changes:

* updated CHANGELOG.md in the repo root to point to changelogs in
subproject folders
* updated dev/release/update_change_log.sh to take subproject as argument
* added dev/update_ballista_versions.py to help update ballista crate versions.

* ignore autogenerated CHANGELOG.md from prettier run

* expand file glob within prettier

'**' pattern is not supported to some of the shells including the one we
use in CI.

* exclude subproject changelog from rat

* update changelog to latest master

* update changelog with updated pr labels

* update datafusion version in readme

* add verify-release-candidate.sh

* update changelog for latest master

* update release email formatting

* update verify-release-candidate.sh permission

* cargo test all in release verify script

.github/workflows/dev.yml

@@ -63,9 +63,12 @@ jobs:
         run: |
           # if you encounter error, try rerun the command below with --write instead of --check
           # and commit the changes
+          #
+          # ignore subproject CHANGELOG.md because they are machine generated
           npx [email protected] --write \
             '{ballista,datafusion,datafusion-examples,docs,python}/**/*.md' \
+            '!{ballista,datafusion,python}/CHANGELOG.md' \
             README.md \
             DEVELOPERS.md \
             'ballista/**/*.{ts,tsx}'
-          git diff --exit-code
+          git diff --exit-code

.github_changelog_generator

@@ -21,10 +21,10 @@
 # point to the old changelog in apache/arrow
 front-matter=For older versions, see [apache/arrow/CHANGELOG.md](https://github.com/apache/arrow/blob/master/CHANGELOG.md)\n
 # some issues are just documentation
-add-sections={"documentation":{"prefix":"**Documentation updates:**","labels":["documentation"]}}
+add-sections={"documentation":{"prefix":"**Documentation updates:**","labels":["documentation"]},"performance":{"prefix":"**Performance improvements:**","labels":["performance"]}}
 # uncomment to not show PRs. TBD if we shown them or not.
 #pull-requests=false
 # so that the component is shown associated with the issue
-issue-line-labels=ballista,datafusion,python
+issue-line-labels=sql
 exclude-labels=development-process,invalid
-breaking_labels=api-change
+breaking-labels=api change

README.md

@@ -130,7 +130,7 @@ To get started, add the following to your `Cargo.toml` file:
 
 ```toml
 [dependencies]
-datafusion = "4.0.0-SNAPSHOT"
+datafusion = "5.0.0"
 ```
 
 ## Using DataFusion as a binary

ballista-examples/Cargo.toml

@@ -18,7 +18,7 @@
 [package]
 name = "ballista-examples"
 description = "Ballista usage examples"
-version = "0.5.0-SNAPSHOT"
+version = "0.5.0"
 homepage = "https://github.com/apache/arrow-datafusion"
 repository = "https://github.com/apache/arrow-datafusion"
 authors = ["Apache Arrow <[email protected]>"]

ballista/rust/client/Cargo.toml

@@ -19,7 +19,7 @@
 name = "ballista"
 description = "Ballista Distributed Compute"
 license = "Apache-2.0"
-version = "0.5.0-SNAPSHOT"
+version = "0.5.0"
 homepage = "https://github.com/apache/arrow-datafusion"
 repository = "https://github.com/apache/arrow-datafusion"
 authors = ["Apache Arrow <[email protected]>"]
@@ -37,4 +37,4 @@ datafusion = { path = "../../../datafusion" }
 
 [features]
 default = []
-standalone = ["ballista-executor", "ballista-scheduler"]
+standalone = ["ballista-executor", "ballista-scheduler"]

ballista/rust/core/Cargo.toml

@@ -19,7 +19,7 @@
 name = "ballista-core"
 description = "Ballista Distributed Compute"
 license = "Apache-2.0"
-version = "0.5.0-SNAPSHOT"
+version = "0.5.0"
 homepage = "https://github.com/apache/arrow-datafusion"
 repository = "https://github.com/apache/arrow-datafusion"
 authors = ["Apache Arrow <[email protected]>"]

ballista/rust/executor/Cargo.toml

@@ -19,7 +19,7 @@
 name = "ballista-executor"
 description = "Ballista Distributed Compute - Executor"
 license = "Apache-2.0"
-version = "0.5.0-SNAPSHOT"
+version = "0.5.0"
 homepage = "https://github.com/apache/arrow-datafusion"
 repository = "https://github.com/apache/arrow-datafusion"
 authors = ["Apache Arrow <[email protected]>"]

ballista/rust/scheduler/Cargo.toml

@@ -19,7 +19,7 @@
 name = "ballista-scheduler"
 description = "Ballista Distributed Compute - Scheduler"
 license = "Apache-2.0"
-version = "0.5.0-SNAPSHOT"
+version = "0.5.0"
 homepage = "https://github.com/apache/arrow-datafusion"
 repository = "https://github.com/apache/arrow-datafusion"
 authors = ["Apache Arrow <[email protected]>"]

datafusion/Cargo.toml

@@ -18,7 +18,7 @@
 [package]
 name = "datafusion"
 description = "DataFusion is an in-memory query engine that uses Apache Arrow as the memory model"
-version = "4.0.0-SNAPSHOT"
+version = "5.0.0"
 homepage = "https://github.com/apache/arrow-datafusion"
 repository = "https://github.com/apache/arrow-datafusion"
 readme = "../README.md"

dev/release/create-tarball.sh

@@ -76,16 +76,21 @@ cat <<MAIL
 To: [email protected]
 Subject: [VOTE][RUST][Datafusion] Release Apache Arrow Datafusion ${tag} RC${rc}
 Hi,
-I would like to propose a release of Apache Arrow Datafusion Implementation, version ${tag}.
+
+I would like to propose a release of Apache Arrow Datafusion Implementation,
+version ${tag}.
+
 This release candidate is based on commit: ${release_hash} [1]
 The proposed release tarball and signatures are hosted at [2].
 The changelog is located at [3].
-Please download, verify checksums and signatures, run the unit tests,
-and vote on the release.
-The vote will be open for at least 72 hours.
+
+Please download, verify checksums and signatures, run the unit tests, and vote
+on the release. The vote will be open for at least 72 hours.
+
 [ ] +1 Release this as Apache Arrow Datafusion ${version}
 [ ] +0
 [ ] -1 Do not release this as Apache Arrow Datafusion ${version} because...
+
 [1]: https://github.com/apache/arrow-datafusion/tree/${release_hash}
 [2]: ${url}
 [3]: https://github.com/apache/arrow-datafusion/blob/${release_hash}/CHANGELOG.md

dev/release/rat_exclude_files.txt

@@ -14,6 +14,9 @@ ci/etc/rprofile
 ci/etc/*.patch
 ci/vcpkg/*.patch
 CHANGELOG.md
+datafusion/CHANGELOG.md
+ballista/CHANGELOG.md
+python/CHANGELOG.md
 dev/requirements*.txt
 dev/archery/MANIFEST.in
 dev/archery/requirements*.txt

dev/release/update_change_log-all.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Usage:
+# CHANGELOG_GITHUB_TOKEN=<TOKEN> ./update_change_log-datafusion.sh
+
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
+
+${SOURCE_DIR}/update_change_log-datafusion.sh
+${SOURCE_DIR}/update_change_log-ballista.sh
+${SOURCE_DIR}/update_change_log-python.sh

dev/release/update_change_log-ballista.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Usage:
+# CHANGELOG_GITHUB_TOKEN=<TOKEN> ./update_change_log-ballista.sh
+
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
+
+CURRENT_VER=$(grep version "${SOURCE_TOP_DIR}/ballista/rust/client/Cargo.toml" | head -n 1 | awk '{print $3}' | tr -d '"')
+${SOURCE_DIR}/update_change_log.sh ballista 4.0.0 "ballista-${CURRENT_VER}"

dev/release/update_change_log-datafusion.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Usage:
+# CHANGELOG_GITHUB_TOKEN=<TOKEN> ./update_change_log-datafusion.sh
+
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
+
+CURRENT_VER=$(grep version "${SOURCE_TOP_DIR}/datafusion/Cargo.toml" | head -n 1 | awk '{print $3}' | tr -d '"')
+${SOURCE_DIR}/update_change_log.sh datafusion 4.0.0 "${CURRENT_VER}"

dev/release/update_change_log-python.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Usage:
+# CHANGELOG_GITHUB_TOKEN=<TOKEN> ./update_change_log-python.sh
+
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
+
+CURRENT_VER=$(grep version "${SOURCE_TOP_DIR}/python/Cargo.toml" | head -n 1 | awk '{print $3}' | tr -d '"')
+${SOURCE_DIR}/update_change_log.sh python 4.0.0 "python-${CURRENT_VER}"

dev/release/update_change_log.sh

@@ -27,21 +27,54 @@
 # arrow-datafusion/.github_changelog_generator
 #
 # Usage:
-# CHANGELOG_GITHUB_TOKEN=<TOKEN> ./update_change_log.sh
+# CHANGELOG_GITHUB_TOKEN=<TOKEN> ./update_change_log.sh <PROJECT> <FROM_VER> <TO_VER>
 
 set -e
 
 SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
 
+if [[ "$#" -ne 3 ]]; then
+    echo "USAGE: $0 PROJECT FROM_VER TO_VER"
+    exit 1
+fi
+
+PROJECT=$1
+FROM_VER=$2
+TO_VER=$3
+OUTPUT_PATH="${PROJECT}/CHANGELOG.md"
+
 pushd ${SOURCE_TOP_DIR}
 docker run -it --rm \
     -e CHANGELOG_GITHUB_TOKEN=$CHANGELOG_GITHUB_TOKEN \
     -v "$(pwd)":/usr/local/src/your-app \
     githubchangeloggenerator/github-changelog-generator \
     --user apache \
     --project arrow-datafusion \
-    --since-tag 4.0.0 \
-    --future-release 5.0.0
+    --since-tag "${FROM_VER}" \
+    --include-labels "${PROJECT}" \
+    --output "${OUTPUT_PATH}" \
+    --future-release "${TO_VER}"
+
+sed -i "s/\\\n/\n\n/" "${OUTPUT_PATH}"
+
+echo '<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
 
-sed -i "s/\\\n/\n\n/" CHANGELOG.md
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+' | cat - "${OUTPUT_PATH}" > "${OUTPUT_PATH}".tmp
+mv "${OUTPUT_PATH}".tmp "${OUTPUT_PATH}"