Skip to content

Commit 435c336

Browse files
m1a2stm1a2st
committed
Merge branch 'trunk' into KAFKA-18180 
2 parents a35fe88 + 369b8b5 commit 435c336

380 files changed

Lines changed: 10942 additions & 5198 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/scripts/develocity_reports.py‎

Lines changed: 954 additions & 0 deletions
Large diffs are not rendered by default.

‎.github/scripts/requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,5 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515
PyYAML~=6.0
16+
pytz==2024.2
17+
requests==2.32.3

‎.github/workflows/docker_scan.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ jobs:
2626
strategy:
2727
matrix:
2828
# This is an array of supported tags. Make sure this array only contains the supported tags
29-
supported_image_tag: ['latest', '3.7.1', "3.8.0", "3.8.1", "3.9.0"]
29+
supported_image_tag: ['latest', '3.7.2', '3.8.1', '3.9.0']
3030
steps:
3131
- name: Run CVE scan
3232
uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8 # v0.24.0

storage/src/test/resources/log4j.properties renamed to .github/workflows/generate-reports.yml

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,32 @@
1212
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
15-
log4j.rootLogger=OFF, stdout
1615

17-
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
18-
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
19-
log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n
16+
name: Generate Reports
17+
on:
18+
workflow_dispatch: # Let us run manually
2019

21-
log4j.appender.fileAppender=org.apache.log4j.RollingFileAppender
22-
log4j.appender.fileAppender.layout=org.apache.log4j.PatternLayout
23-
log4j.appender.fileAppender.layout.ConversionPattern=%d [%t] %-5p %c %x - %m%n
24-
log4j.appender.fileAppender.File=storage.log
20+
schedule:
21+
- cron: '0 6 * * *' # Run daily at 6am UTC
2522

26-
log4j.logger.org.apache.kafka.server.log.remote.storage=INFO
27-
log4j.logger.org.apache.kafka.server.log.remote.metadata.storage=INFO
28-
log4j.logger.kafka.log.remote=INFO
23+
24+
jobs:
25+
flaky-test-report:
26+
name: Flaky Test Report
27+
permissions:
28+
contents: read
29+
runs-on: ubuntu-latest
30+
steps:
31+
- name: Env
32+
run: printenv
33+
env:
34+
GITHUB_CONTEXT: ${{ toJson(github) }}
35+
- name: Checkout code
36+
uses: actions/checkout@v4
37+
- name: Setup Python
38+
uses: ./.github/actions/setup-python
39+
- name: Run Report
40+
env:
41+
GE_ACCESS_TOKEN: ${{ secrets.GE_ACCESS_TOKEN }}
42+
run: |
43+
python ./.github/scripts/develocity_reports.py >> $GITHUB_STEP_SUMMARY

‎.github/workflows/pr-reviewed.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ jobs:
4040
name: pr-number.txt
4141
- name: Remove label
4242
uses: actions/github-script@v7
43+
continue-on-error: true
4344
with:
4445
github-token: ${{ secrets.GITHUB_TOKEN }}
4546
script: |

‎LICENSE-binary‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,7 @@ jackson-annotations-2.16.2
220220
jackson-core-2.16.2
221221
jackson-databind-2.16.2
222222
jackson-dataformat-csv-2.16.2
223+
jackson-dataformat-yaml-2.16.2
223224
jackson-datatype-jdk8-2.16.2
224225
jackson-jaxrs-base-2.16.2
225226
jackson-jaxrs-json-provider-2.16.2
@@ -239,6 +240,11 @@ jetty-servlets-9.4.56.v20240826
239240
jetty-util-9.4.56.v20240826
240241
jetty-util-ajax-9.4.56.v20240826
241242
jose4j-0.9.4
243+
log4j-api-2.24.1
244+
log4j-core-2.24.1
245+
log4j-core-test-2.24.1
246+
log4j-slf4j-impl-2.24.1
247+
log4j-1.2-api-2.24.1
242248
lz4-java-1.8.0
243249
maven-artifact-3.9.6
244250
metrics-core-4.1.12.1
@@ -254,7 +260,6 @@ netty-transport-native-epoll-4.1.115.Final
254260
netty-transport-native-unix-common-4.1.115.Final
255261
opentelemetry-proto-1.0.0-alpha
256262
plexus-utils-3.5.1
257-
reload4j-1.2.25
258263
rocksdbjni-7.9.2
259264
scala-library-2.13.15
260265
scala-logging_2.13-3.9.5
@@ -302,7 +307,6 @@ see: licenses/CDDL+GPL-1.1
302307
javax.activation-api-1.2.0
303308
javax.annotation-api-1.3.2
304309
javax.servlet-api-3.1.0
305-
javax.ws.rs-api-2.1.1
306310
jaxb-api-2.3.1
307311
activation-1.1.1
308312

@@ -313,7 +317,6 @@ argparse4j-0.7.0, see: licenses/argparse-MIT
313317
classgraph-4.8.173, see: licenses/classgraph-MIT
314318
jopt-simple-5.0.4, see: licenses/jopt-simple-MIT
315319
slf4j-api-1.7.36, see: licenses/slf4j-MIT
316-
slf4j-reload4j-1.7.36, see: licenses/slf4j-MIT
317320
pcollections-4.0.1, see: licenses/pcollections-MIT
318321

319322
---------------------------------------

‎README.md‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,10 @@ Follow instructions in https://kafka.apache.org/quickstart
5050
./gradlew clients:test --tests org.apache.kafka.clients.MetadataTest.testTimeToNextUpdate
5151

5252
### Running a particular unit/integration test with log4j output ###
53-
By default, there will be only small number of logs output while testing. You can adjust it by changing the `log4j.properties` file in the module's `src/test/resources` directory.
53+
By default, there will be only small number of logs output while testing. You can adjust it by changing the `log4j2.yml` file in the module's `src/test/resources` directory.
5454

55-
For example, if you want to see more logs for clients project tests, you can modify [the line](https://github.com/apache/kafka/blob/trunk/clients/src/test/resources/log4j.properties#L21) in `clients/src/test/resources/log4j.properties`
56-
to `log4j.logger.org.apache.kafka=INFO` and then run:
55+
For example, if you want to see more logs for clients project tests, you can modify [the line](https://github.com/apache/kafka/blob/trunk/clients/src/test/resources/log4j2.yml#L35) in `clients/src/test/resources/log4j2.yml`
56+
to `level: INFO` and then run:
5757

5858
./gradlew cleanTest clients:test --tests NetworkClientTest
5959

@@ -121,6 +121,8 @@ Streams has multiple sub-projects, but you can run all the tests:
121121
./gradlew tasks
122122

123123
### Building IDE project ####
124+
*Note Please ensure that JDK17 is used when developing Kafka.*
125+
124126
*Note that this is not strictly necessary (IntelliJ IDEA has good built-in support for Gradle projects, for example).*
125127

126128
./gradlew eclipse
@@ -130,6 +132,11 @@ The `eclipse` task has been configured to use `${project_dir}/build_eclipse` as
130132
build directory (`${project_dir}/bin`) clashes with Kafka's scripts directory and we don't use Gradle's build directory
131133
to avoid known issues with this configuration.
132134

135+
IntelliJ Language Level awareness:
136+
137+
IntelliJ will automatically check Java syntax and compatibility for each module, even if the Java version is not
138+
explicitly set in the Structure > Project Settings > Modules.
139+
133140
### Publishing the streams quickstart archetype artifact to maven ###
134141
For the Streams archetype project, one cannot use gradle to upload to maven; instead the `mvn deploy` command needs to be called at the quickstart folder:
135142

‎bin/connect-distributed.sh‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,12 @@ fi
2222

2323
base_dir=$(dirname $0)
2424

25-
if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
26-
export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/connect-log4j.properties"
25+
if [ -z "$KAFKA_LOG4J_OPTS" ]; then
26+
export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/connect-log4j2.yaml"
27+
elif echo "$KAFKA_LOG4J_OPTS" | grep -qE "log4j\.[^[:space:]]+$"; then
28+
echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended. >&2
29+
echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. >&2
30+
echo You can also use the \$KAFKA_HOME/config/connect-log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. >&2
2731
fi
2832

2933
if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then

‎bin/connect-mirror-maker.sh‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,12 @@ fi
2222

2323
base_dir=$(dirname $0)
2424

25-
if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
26-
export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/connect-log4j.properties"
25+
if [ -z "$KAFKA_LOG4J_OPTS" ]; then
26+
export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/connect-log4j2.yaml"
27+
elif echo "$KAFKA_LOG4J_OPTS" | grep -qE "log4j\.[^[:space:]]+$"; then
28+
echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended. >&2
29+
echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. >&2
30+
echo You can also use the \$KAFKA_HOME/config/connect-log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. >&2
2731
fi
2832

2933
if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then

‎bin/connect-standalone.sh‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,12 @@ fi
2222

2323
base_dir=$(dirname $0)
2424

25-
if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
26-
export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/connect-log4j.properties"
25+
if [ -z "$KAFKA_LOG4J_OPTS" ]; then
26+
export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/connect-log4j2.yaml"
27+
elif echo "$KAFKA_LOG4J_OPTS" | grep -qE "log4j\.[^[:space:]]+$"; then
28+
echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended. >&2
29+
echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. >&2
30+
echo You can also use the \$KAFKA_HOME/config/connect-log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. >&2
2731
fi
2832

2933
if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then

0 commit comments

Comments
 (0)