|
| 1 | +package consume |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "encoding/json" |
| 6 | + "fmt" |
| 7 | + "os" |
| 8 | + "strconv" |
| 9 | + "sync" |
| 10 | + |
| 11 | + "github.com/Shopify/sarama" |
| 12 | + "github.com/fgrosse/cli" |
| 13 | + "github.com/fgrosse/kafkactl/pkg" |
| 14 | + "github.com/pkg/errors" |
| 15 | + "github.com/spf13/cobra" |
| 16 | + "github.com/spf13/viper" |
| 17 | +) |
| 18 | + |
| 19 | +func (cmd *command) ConsumeCmd() *cobra.Command { |
| 20 | + produceCmd := &cobra.Command{ |
| 21 | + Use: "consume <topic>", |
| 22 | + Args: cobra.ExactArgs(1), |
| 23 | + Short: "Consume messages from a Kafka topic and print them to stdout", |
| 24 | + Long: `Consume messages from a Kafka topic and print them to stdout. |
| 25 | +
|
| 26 | +By default, the --output flag is set to "raw" which means that the command will |
| 27 | +only print the message values followed by a newline. You can set --output=json |
| 28 | +in order to print each consumed message as a JSON object which will contain the |
| 29 | +partition and offset information in addition to the message value. |
| 30 | +
|
| 31 | +Values will automatically be decoded using the topic schema configuration from the |
| 32 | +kafkactl configuration file (e.g. to decode proto messages and print them as JSON). |
| 33 | +If no configuration matches the topic name, message values will be assumed to be |
| 34 | +unicode strings. |
| 35 | +
|
| 36 | +This command will block as long as it is connected to Kafka. You can stop reading |
| 37 | +messages by sending SIGINT, SIGQUIT or SIGTERM to the process (e.g. by pressing ctrl+c). |
| 38 | +`, |
| 39 | + Example: ` |
| 40 | + # Read and print all messages from "example-topic" without joining a consumer group |
| 41 | + kafkactl consume example-topic |
| 42 | + |
| 43 | + # Read messages only from a specific partition |
| 44 | + kafkactl consume example-topic --partition=1 |
| 45 | + |
| 46 | + # Join the "test" consumer group and print all messages that are assigned to this member |
| 47 | + kafkactl consume example-topic --group=test |
| 48 | +`, |
| 49 | + RunE: func(_ *cobra.Command, args []string) error { |
| 50 | + ctx := cli.Context() |
| 51 | + topic := args[0] |
| 52 | + partition := viper.GetInt32("partition") |
| 53 | + offset := viper.GetString("offset") |
| 54 | + outputEncoding := viper.GetString("output") |
| 55 | + return cmd.consume(ctx, topic, partition, offset, outputEncoding) |
| 56 | + }, |
| 57 | + } |
| 58 | + |
| 59 | + flags := produceCmd.Flags() |
| 60 | + flags.Int32("partition", -1, "Kafka topic partition. -1 means all partitions") |
| 61 | + flags.String("offset", "newest", `either "oldest", "newest" or an integer`) |
| 62 | + flags.StringP("output", "o", "raw", "output format. One of raw|json. See --help output for more information") |
| 63 | + // TODO: support joining a consumer group |
| 64 | + |
| 65 | + return produceCmd |
| 66 | +} |
| 67 | + |
| 68 | +func (cmd *command) consume(ctx context.Context, topic string, partition int32, offsetStr, outputEncoding string) error { |
| 69 | + var offset int64 |
| 70 | + switch offsetStr { |
| 71 | + case "oldest", "first": |
| 72 | + offset = sarama.OffsetOldest |
| 73 | + case "newest", "last": |
| 74 | + offset = sarama.OffsetNewest |
| 75 | + default: |
| 76 | + n, err := strconv.Atoi(offsetStr) |
| 77 | + if err != nil { |
| 78 | + return fmt.Errorf("failed to parse --offset as integer: %w", err) |
| 79 | + } |
| 80 | + offset = int64(n) |
| 81 | + } |
| 82 | + |
| 83 | + conf := cmd.Configuration() |
| 84 | + dec, err := pkg.NewTopicDecoder(topic, *conf) |
| 85 | + if err != nil { |
| 86 | + return err |
| 87 | + } |
| 88 | + |
| 89 | + messages, err := cmd.simpleConsumer(ctx, topic, partition, offset) |
| 90 | + if err != nil { |
| 91 | + return err |
| 92 | + } |
| 93 | + |
| 94 | + for msg := range messages { |
| 95 | + decoded, err := dec.Decode(msg) |
| 96 | + if err != nil { |
| 97 | + return fmt.Errorf("failed to decode message from Kafka: %w", err) |
| 98 | + } |
| 99 | + |
| 100 | + switch outputEncoding { |
| 101 | + case "raw": |
| 102 | + fmt.Fprintln(os.Stdout, decoded.Value) |
| 103 | + case "json": |
| 104 | + val, err := json.Marshal(decoded) |
| 105 | + if err != nil { |
| 106 | + return err |
| 107 | + } |
| 108 | + fmt.Fprintln(os.Stdout, string(val)) |
| 109 | + } |
| 110 | + |
| 111 | + } |
| 112 | + |
| 113 | + return nil |
| 114 | +} |
| 115 | + |
| 116 | +func (cmd *command) simpleConsumer(ctx context.Context, topic string, partition int32, offset int64) (<-chan *sarama.ConsumerMessage, error) { |
| 117 | + conf := cmd.SaramaConfig() |
| 118 | + conf.Consumer.Return.Errors = false // TODO |
| 119 | + |
| 120 | + brokers := cmd.Configuration().Brokers() |
| 121 | + c, err := sarama.NewConsumer(brokers, conf) |
| 122 | + if err != nil { |
| 123 | + return nil, fmt.Errorf("failed to create consumer: %w", err) |
| 124 | + } |
| 125 | + |
| 126 | + if partition >= 0 { |
| 127 | + return cmd.consumeSinglePartition(ctx, c, topic, partition, offset) |
| 128 | + } |
| 129 | + |
| 130 | + return cmd.consumeAllPartitions(ctx, c, topic, offset) |
| 131 | +} |
| 132 | + |
| 133 | +func (cmd *command) consumeSinglePartition(ctx context.Context, c sarama.Consumer, topic string, partition int32, offset int64) (<-chan *sarama.ConsumerMessage, error) { |
| 134 | + con, err := c.ConsumePartition(topic, partition, offset) |
| 135 | + if err != nil { |
| 136 | + return nil, errors.Wrap(err, "failed to consume topic partition") |
| 137 | + } |
| 138 | + |
| 139 | + go func() { |
| 140 | + <-ctx.Done() |
| 141 | + con.AsyncClose() |
| 142 | + }() |
| 143 | + |
| 144 | + cmd.debug.Printf("Consuming topic %q partition %d starting at offset %d", topic, partition, offset) |
| 145 | + return con.Messages(), nil |
| 146 | +} |
| 147 | + |
| 148 | +func (cmd *command) consumeAllPartitions(ctx context.Context, c sarama.Consumer, topic string, offset int64) (<-chan *sarama.ConsumerMessage, error) { |
| 149 | + partitions, err := c.Partitions(topic) |
| 150 | + if err != nil { |
| 151 | + return nil, errors.Wrap(err, "get partitions") |
| 152 | + } |
| 153 | + |
| 154 | + var wg sync.WaitGroup |
| 155 | + messages := make(chan *sarama.ConsumerMessage, len(partitions)) |
| 156 | + |
| 157 | + for _, partition := range partitions { |
| 158 | + con, err := c.ConsumePartition(topic, partition, offset) |
| 159 | + if err != nil { |
| 160 | + return nil, errors.Wrapf(err, "consume partition %d", partition) |
| 161 | + } |
| 162 | + |
| 163 | + output := func(partitionMessages <-chan *sarama.ConsumerMessage) { |
| 164 | + defer wg.Done() |
| 165 | + for msg := range partitionMessages { |
| 166 | + select { |
| 167 | + case messages <- msg: |
| 168 | + case <-ctx.Done(): |
| 169 | + return |
| 170 | + } |
| 171 | + } |
| 172 | + } |
| 173 | + |
| 174 | + wg.Add(1) |
| 175 | + go output(con.Messages()) |
| 176 | + |
| 177 | + // Close this partition consumer when the context is done. |
| 178 | + go func() { |
| 179 | + <-ctx.Done() |
| 180 | + con.AsyncClose() |
| 181 | + }() |
| 182 | + } |
| 183 | + |
| 184 | + // When all individual partition consumers are done, close the messages channel. |
| 185 | + go func() { |
| 186 | + wg.Wait() |
| 187 | + close(messages) |
| 188 | + }() |
| 189 | + |
| 190 | + return messages, nil |
| 191 | +} |
0 commit comments