ClickHouse · vdimir · Aug 24, 2021 · Apr 14, 2021 · Apr 14, 2021 · May 25, 2021
diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
@@ -393,6 +393,9 @@ function run_tests
         01853_s2_cells_intersect
         01854_s2_cap_contains
         01854_s2_cap_union
+
+        # needs s3
+        01944_insert_partition_by
     )
 
     time clickhouse-test --hung-check -j 8 --order=random --use-skip-list \

diff --git a/src/Common/isValidUTF8.cpp b/src/Common/isValidUTF8.cpp
@@ -0,0 +1,131 @@
+#include <Common/isValidUTF8.h>
+#include <cstring>
+
+/// inspired by https://github.com/cyb70289/utf8/
+
+/*
+MIT License
+
+Copyright (c) 2019 Yibo Cai
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+/*
+* http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94
+*
+* Table 3-7. Well-Formed UTF-8 Byte Sequences
+*
+* +--------------------+------------+-------------+------------+-------------+
+* | Code Points        | First Byte | Second Byte | Third Byte | Fourth Byte |
+* +--------------------+------------+-------------+------------+-------------+
+* | U+0000..U+007F     | 00..7F     |             |            |             |
+* +--------------------+------------+-------------+------------+-------------+
+* | U+0080..U+07FF     | C2..DF     | 80..BF      |            |             |
+* +--------------------+------------+-------------+------------+-------------+
+* | U+0800..U+0FFF     | E0         | A0..BF      | 80..BF     |             |
+* +--------------------+------------+-------------+------------+-------------+
+* | U+1000..U+CFFF     | E1..EC     | 80..BF      | 80..BF     |             |
+* +--------------------+------------+-------------+------------+-------------+
+* | U+D000..U+D7FF     | ED         | 80..9F      | 80..BF     |             |
+* +--------------------+------------+-------------+------------+-------------+
+* | U+E000..U+FFFF     | EE..EF     | 80..BF      | 80..BF     |             |
+* +--------------------+------------+-------------+------------+-------------+
+* | U+10000..U+3FFFF   | F0         | 90..BF      | 80..BF     | 80..BF      |
+* +--------------------+------------+-------------+------------+-------------+
+* | U+40000..U+FFFFF   | F1..F3     | 80..BF      | 80..BF     | 80..BF      |
+* +--------------------+------------+-------------+------------+-------------+
+* | U+100000..U+10FFFF | F4         | 80..8F      | 80..BF     | 80..BF      |
+* +--------------------+------------+-------------+------------+-------------+
+*/
+namespace DB
+{
+
+namespace UTF8
+{
+
+UInt8 isValidUTF8(const UInt8 * data, UInt64 len)
+{
+    while (len)
+    {
+        int bytes;
+        const UInt8 byte1 = data[0];
+        /* 00..7F */
+        if (byte1 <= 0x7F)
+        {
+            bytes = 1;
+        }
+        /* C2..DF, 80..BF */
+        else if (len >= 2 && byte1 >= 0xC2 && byte1 <= 0xDF && static_cast<Int8>(data[1]) <= static_cast<Int8>(0xBF))
+        {
+            bytes = 2;
+        }
+        else if (len >= 3)
+        {
+            const UInt8 byte2 = data[1];
+            bool byte2_ok = static_cast<Int8>(byte2) <= static_cast<Int8>(0xBF);
+            bool byte3_ok = static_cast<Int8>(data[2]) <= static_cast<Int8>(0xBF);
+
+            if (byte2_ok && byte3_ok &&
+                /* E0, A0..BF, 80..BF */
+                ((byte1 == 0xE0 && byte2 >= 0xA0) ||
+                 /* E1..EC, 80..BF, 80..BF */
+                 (byte1 >= 0xE1 && byte1 <= 0xEC) ||
+                 /* ED, 80..9F, 80..BF */
+                 (byte1 == 0xED && byte2 <= 0x9F) ||
+                 /* EE..EF, 80..BF, 80..BF */
+                 (byte1 >= 0xEE && byte1 <= 0xEF)))
+            {
+                bytes = 3;
+            }
+            else if (len >= 4)
+            {
+                bool byte4_ok = static_cast<Int8>(data[3]) <= static_cast<Int8>(0xBF);
+                if (byte2_ok && byte3_ok && byte4_ok &&
+                    /* F0, 90..BF, 80..BF, 80..BF */
+                    ((byte1 == 0xF0 && byte2 >= 0x90) ||
+                     /* F1..F3, 80..BF, 80..BF, 80..BF */
+                     (byte1 >= 0xF1 && byte1 <= 0xF3) ||
+                     /* F4, 80..8F, 80..BF, 80..BF */
+                     (byte1 == 0xF4 && byte2 <= 0x8F)))
+                {
+                    bytes = 4;
+                }
+                else
+                {
+                    return false;
+                }
+            }
+            else
+            {
+                return false;
+            }
+        }
+        else
+        {
+            return false;
+        }
+        len -= bytes;
+        data += bytes;
+    }
+    return true;
+}
+
+}
+}
diff --git a/src/Common/isValidUTF8.h b/src/Common/isValidUTF8.h
@@ -0,0 +1,10 @@
+#pragma once
+
+#include <common/types.h>
+
+namespace DB::UTF8
+{
+
+UInt8 isValidUTF8(const UInt8 * data, UInt64 len);
+
+}
diff --git a/src/Common/ya.make b/src/Common/ya.make
@@ -116,6 +116,7 @@ SRCS(
     hasLinuxCapability.cpp
     hex.cpp
     isLocalAddress.cpp
+    isValidUTF8.cpp
     malloc.cpp
     memory.cpp
     new_delete.cpp

diff --git a/src/Functions/isValidUTF8.cpp b/src/Functions/isValidUTF8.cpp
@@ -1,14 +1,7 @@
 #include <DataTypes/DataTypeString.h>
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionStringOrArrayToT.h>
-
-#include <cstring>
-
-#ifdef __SSE4_1__
-#    include <emmintrin.h>
-#    include <smmintrin.h>
-#    include <tmmintrin.h>
-#endif
+#include <Common/isValidUTF8.h>
 
 namespace DB
 {
@@ -71,75 +64,8 @@ SOFTWARE.
  * +--------------------+------------+-------------+------------+-------------+
  */
 
-    static inline UInt8 isValidUTF8Naive(const UInt8 * data, UInt64 len)
-    {
-        while (len)
-        {
-            int bytes;
-            const UInt8 byte1 = data[0];
-            /* 00..7F */
-            if (byte1 <= 0x7F)
-            {
-                bytes = 1;
-            }
-            /* C2..DF, 80..BF */
-            else if (len >= 2 && byte1 >= 0xC2 && byte1 <= 0xDF && static_cast<Int8>(data[1]) <= static_cast<Int8>(0xBF))
-            {
-                bytes = 2;
-            }
-            else if (len >= 3)
-            {
-                const UInt8 byte2 = data[1];
-                bool byte2_ok = static_cast<Int8>(byte2) <= static_cast<Int8>(0xBF);
-                bool byte3_ok = static_cast<Int8>(data[2]) <= static_cast<Int8>(0xBF);
-
-                if (byte2_ok && byte3_ok &&
-                    /* E0, A0..BF, 80..BF */
-                    ((byte1 == 0xE0 && byte2 >= 0xA0) ||
-                     /* E1..EC, 80..BF, 80..BF */
-                     (byte1 >= 0xE1 && byte1 <= 0xEC) ||
-                     /* ED, 80..9F, 80..BF */
-                     (byte1 == 0xED && byte2 <= 0x9F) ||
-                     /* EE..EF, 80..BF, 80..BF */
-                     (byte1 >= 0xEE && byte1 <= 0xEF)))
-                {
-                    bytes = 3;
-                }
-                else if (len >= 4)
-                {
-                    bool byte4_ok = static_cast<Int8>(data[3]) <= static_cast<Int8>(0xBF);
-                    if (byte2_ok && byte3_ok && byte4_ok &&
-                        /* F0, 90..BF, 80..BF, 80..BF */
-                        ((byte1 == 0xF0 && byte2 >= 0x90) ||
-                         /* F1..F3, 80..BF, 80..BF, 80..BF */
-                         (byte1 >= 0xF1 && byte1 <= 0xF3) ||
-                         /* F4, 80..8F, 80..BF, 80..BF */
-                         (byte1 == 0xF4 && byte2 <= 0x8F)))
-                    {
-                        bytes = 4;
-                    }
-                    else
-                    {
-                        return false;
-                    }
-                }
-                else
-                {
-                    return false;
-                }
-            }
-            else
-            {
-                return false;
-            }
-            len -= bytes;
-            data += bytes;
-        }
-        return true;
-    }
-
 #ifndef __SSE4_1__
-    static inline UInt8 isValidUTF8(const UInt8 * data, UInt64 len) { return isValidUTF8Naive(data, len); }
+    static inline UInt8 isValidUTF8(const UInt8 * data, UInt64 len) { return DB::UTF8::isValidUTF8(data, len); }
 #else
     static inline UInt8 isValidUTF8(const UInt8 * data, UInt64 len)
     {

diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp
@@ -2,9 +2,10 @@
 
 #if USE_AWS_S3
 
+#    include <IO/S3Common.h>
+
 #    include <Common/quoteString.h>
 
-#    include <IO/S3Common.h>
 #    include <IO/WriteBufferFromString.h>
 #    include <Storages/StorageS3Settings.h>
 
@@ -617,7 +618,7 @@ namespace S3
         storage_name = S3;
 
         if (uri.getHost().empty())
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Host is empty in S3 URI: {}", uri.toString());
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Host is empty in S3 URI.");
 
         String name;
         String endpoint_authority_from_uri;
@@ -626,12 +627,7 @@ namespace S3
         {
             is_virtual_hosted_style = true;
             endpoint = uri.getScheme() + "://" + name + endpoint_authority_from_uri;
-
-            /// S3 specification requires at least 3 and at most 63 characters in bucket name.
-            /// https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html
-            if (bucket.length() < 3 || bucket.length() > 63)
-                throw Exception(ErrorCodes::BAD_ARGUMENTS,
-                    "Bucket name length is out of bounds in virtual hosted style S3 URI: {} ({})", quoteString(bucket), uri.toString());
+            validateBucket(bucket, uri);
 
             if (!uri.getPath().empty())
             {
@@ -642,7 +638,7 @@ namespace S3
             boost::to_upper(name);
             if (name != S3 && name != COS)
             {
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Object storage system name is unrecognized in virtual hosted style S3 URI: {} ({})", quoteString(name), uri.toString());
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Object storage system name is unrecognized in virtual hosted style S3 URI: {}", quoteString(name));
             }
             if (name == S3)
             {
@@ -657,14 +653,19 @@ namespace S3
         {
             is_virtual_hosted_style = false;
             endpoint = uri.getScheme() + "://" + uri.getAuthority();
-
-            /// S3 specification requires at least 3 and at most 63 characters in bucket name.
-            /// https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html
-            if (bucket.length() < 3 || bucket.length() > 63)
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bucket name length is out of bounds in virtual hosted style S3 URI: {} ({})", quoteString(bucket), uri.toString());
+            validateBucket(bucket, uri);
         }
         else
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bucket or key name are invalid in S3 URI: {}", uri.toString());
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bucket or key name are invalid in S3 URI.");
+    }
+
+    void URI::validateBucket(const String & bucket, const Poco::URI & uri)
+    {
+        /// S3 specification requires at least 3 and at most 63 characters in bucket name.
+        /// https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html
+        if (bucket.length() < 3 || bucket.length() > 63)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bucket name length is out of bounds in virtual hosted style S3 URI:     {}{}",
+                            quoteString(bucket), !uri.empty() ? " (" + uri.toString() + ")" : "");
     }
 }
 

diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h
@@ -74,6 +74,8 @@ struct URI
     bool is_virtual_hosted_style;
 
     explicit URI(const Poco::URI & uri_);
+
+    static void validateBucket(const String & bucket, const Poco::URI & uri);
 };
 
 }

diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -37,6 +37,7 @@ namespace DB
 
 namespace ErrorCodes
 {
+    extern const int NOT_IMPLEMENTED;
     extern const int NO_SUCH_COLUMN_IN_TABLE;
     extern const int ILLEGAL_COLUMN;
     extern const int DUPLICATE_COLUMN;
@@ -155,6 +156,9 @@ BlockIO InterpreterInsertQuery::execute()
     BlockIO res;
 
     StoragePtr table = getTable(query);
+    if (query.partition_by && !table->supportsPartitionBy())
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "PARTITION BY clause is not supported by storage");
+
     auto table_lock = table->lockForShare(getContext()->getInitialQueryId(), settings.lock_acquire_timeout);
     auto metadata_snapshot = table->getInMemoryMetadataPtr();
 

diff --git a/src/Parsers/ASTInsertQuery.cpp b/src/Parsers/ASTInsertQuery.cpp
@@ -25,6 +25,11 @@ void ASTInsertQuery::formatImpl(const FormatSettings & settings, FormatState & s
     {
         settings.ostr << (settings.hilite ? hilite_keyword : "") << "FUNCTION ";
         table_function->formatImpl(settings, state, frame);
+        if (partition_by)
+        {
+            settings.ostr << " PARTITION BY ";
+            partition_by->formatImpl(settings, state, frame);
+        }
     }
     else
         settings.ostr << (settings.hilite ? hilite_none : "")

diff --git a/src/Parsers/ASTInsertQuery.h b/src/Parsers/ASTInsertQuery.h
@@ -20,6 +20,7 @@ class ASTInsertQuery : public IAST
     ASTPtr infile;
     ASTPtr watch;
     ASTPtr table_function;
+    ASTPtr partition_by;
     ASTPtr settings_ast;
 
     /// Data to insert
@@ -44,6 +45,7 @@ class ASTInsertQuery : public IAST
         if (select) { res->select = select->clone(); res->children.push_back(res->select); }
         if (watch) { res->watch = watch->clone(); res->children.push_back(res->watch); }
         if (table_function) { res->table_function = table_function->clone(); res->children.push_back(res->table_function); }
+        if (partition_by) { res->partition_by = partition_by->clone(); res->children.push_back(res->partition_by); }
         if (settings_ast) { res->settings_ast = settings_ast->clone(); res->children.push_back(res->settings_ast); }
 
         return res;
-Original file line number
+Diff line change
@@ Expand Up / @@ -74,6 +74,8 @@ struct URI @@
         bool is_virtual_hosted_style;
         explicit URI(const Poco::URI & uri_);
+        static void validateBucket(const String & bucket, const Poco::URI & uri);
     };
     }
@@ Expand Down @@