|
22 | 22 | import org.apache.doris.catalog.Database; |
23 | 23 | import org.apache.doris.catalog.OlapTable; |
24 | 24 | import org.apache.doris.common.AnalysisException; |
| 25 | +import org.apache.doris.common.Config; |
25 | 26 | import org.apache.doris.common.DuplicatedRequestException; |
26 | 27 | import org.apache.doris.common.LabelAlreadyUsedException; |
27 | 28 | import org.apache.doris.common.UserException; |
|
41 | 42 | import org.apache.doris.thrift.TTxnParams; |
42 | 43 | import org.apache.doris.thrift.TUniqueId; |
43 | 44 | import org.apache.doris.transaction.BeginTransactionException; |
| 45 | +import org.apache.doris.transaction.DatabaseTransactionMgr; |
44 | 46 | import org.apache.doris.transaction.GlobalTransactionMgr; |
45 | 47 | import org.apache.doris.transaction.TransactionEntry; |
46 | 48 | import org.apache.doris.transaction.TransactionState; |
@@ -121,53 +123,69 @@ public void beginTxn(long batchId) throws UserException, TException, TimeoutExce |
121 | 123 | + "_batch" + batchId + "_" + currentTime; |
122 | 124 | String targetColumn = Joiner.on(",").join(columns) + "," + DELETE_COLUMN; |
123 | 125 | GlobalTransactionMgr globalTransactionMgr = Catalog.getCurrentGlobalTransactionMgr(); |
124 | | - TransactionEntry txnEntry = txnExecutor.getTxnEntry(); |
125 | | - TTxnParams txnConf = txnEntry.getTxnConf(); |
126 | | - TransactionState.LoadJobSourceType sourceType = TransactionState.LoadJobSourceType.INSERT_STREAMING; |
127 | | - TStreamLoadPutRequest request = null; |
128 | | - try { |
129 | | - long txnId = globalTransactionMgr.beginTransaction(db.getId(), Lists.newArrayList(tbl.getId()), label, |
130 | | - new TransactionState.TxnCoordinator(TransactionState.TxnSourceType.FE, FrontendOptions.getLocalHostAddress()), sourceType, timeoutSecond); |
131 | | - String authCodeUuid = Catalog.getCurrentGlobalTransactionMgr().getTransactionState( |
| 126 | + DatabaseTransactionMgr databaseTransactionMgr = globalTransactionMgr.getDatabaseTransactionMgr(db.getId()); |
| 127 | + if (databaseTransactionMgr.getRunningTxnNums() < Config.max_running_txn_num_per_db) { |
| 128 | + TransactionEntry txnEntry = txnExecutor.getTxnEntry(); |
| 129 | + TTxnParams txnConf = txnEntry.getTxnConf(); |
| 130 | + TransactionState.LoadJobSourceType sourceType = TransactionState.LoadJobSourceType.INSERT_STREAMING; |
| 131 | + TStreamLoadPutRequest request = null; |
| 132 | + try { |
| 133 | + long txnId = globalTransactionMgr.beginTransaction(db.getId(), |
| 134 | + Lists.newArrayList(tbl.getId()), label, |
| 135 | + new TransactionState.TxnCoordinator(TransactionState.TxnSourceType.FE, |
| 136 | + FrontendOptions.getLocalHostAddress()), sourceType, timeoutSecond); |
| 137 | + String authCodeUuid = Catalog.getCurrentGlobalTransactionMgr().getTransactionState( |
132 | 138 | db.getId(), txnId).getAuthCode(); |
133 | | - request = new TStreamLoadPutRequest() |
| 139 | + request = new TStreamLoadPutRequest() |
134 | 140 | .setTxnId(txnId).setDb(txnConf.getDb()).setTbl(txnConf.getTbl()) |
135 | 141 | .setFileType(TFileType.FILE_STREAM).setFormatType(TFileFormatType.FORMAT_CSV_PLAIN) |
136 | 142 | .setThriftRpcTimeoutMs(5000).setLoadId(txnExecutor.getLoadId()) |
137 | 143 | .setMergeType(TMergeType.MERGE).setDeleteCondition(DELETE_CONDITION) |
138 | 144 | .setColumns(targetColumn); |
139 | | - txnConf.setTxnId(txnId).setAuthCodeUuid(authCodeUuid); |
140 | | - txnEntry.setLabel(label); |
141 | | - txnExecutor.setTxnId(txnId); |
142 | | - } catch (DuplicatedRequestException e) { |
143 | | - LOG.warn("duplicate request for sync channel. channel: {}, request id: {}, txn: {}, table: {}", |
| 145 | + txnConf.setTxnId(txnId).setAuthCodeUuid(authCodeUuid); |
| 146 | + txnEntry.setLabel(label); |
| 147 | + txnExecutor.setTxnId (txnId); |
| 148 | + } catch (DuplicatedRequestException e) { |
| 149 | + LOG.warn ("duplicate request for sync channel. channel: {}, request id: {}, txn: {}, table: {}", |
144 | 150 | id, e.getDuplicatedRequestId(), e.getTxnId(), targetTable); |
145 | | - txnExecutor.setTxnId(e.getTxnId()); |
146 | | - } catch (LabelAlreadyUsedException e) { |
147 | | - // this happens when channel re-consume same batch, we should just pass through it without begin a new txn |
148 | | - LOG.warn("Label already used in channel {}, label: {}, table: {}, batch: {}", id, label, targetTable, batchId); |
149 | | - return; |
150 | | - } catch (AnalysisException | BeginTransactionException e) { |
151 | | - LOG.warn("encounter an error when beginning txn in channel {}, table: {}", id, targetTable); |
152 | | - throw e; |
153 | | - } catch (UserException e) { |
154 | | - LOG.warn("encounter an error when creating plan in channel {}, table: {}", id, targetTable); |
155 | | - throw e; |
156 | | - } |
157 | | - try { |
158 | | - // async exec begin transaction |
159 | | - long txnId = txnExecutor.getTxnId(); |
160 | | - if (txnId != -1L) { |
161 | | - this.txnExecutor.beginTransaction(request); |
162 | | - LOG.info("begin txn in channel {}, table: {}, label:{}, txn id: {}", id, targetTable, label, txnExecutor.getTxnId()); |
| 151 | + txnExecutor.setTxnId(e.getTxnId()); |
| 152 | + } catch (LabelAlreadyUsedException e) { |
| 153 | + // this happens when channel re-consume same batch, |
| 154 | + // we should just pass through it without begin a new txn |
| 155 | + LOG.warn ("Label already used in channel {}, label: {}, table: {}, batch: {}", |
| 156 | + id, label, targetTable, batchId); |
| 157 | + return; |
| 158 | + } catch (AnalysisException | BeginTransactionException e) { |
| 159 | + LOG.warn ("encounter an error when beginning txn in channel {}, table: {}", |
| 160 | + id, targetTable); |
| 161 | + throw e; |
| 162 | + } catch (UserException e) { |
| 163 | + LOG.warn ("encounter an error when creating plan in channel {}, table: {}", |
| 164 | + id, targetTable); |
| 165 | + throw e; |
163 | 166 | } |
164 | | - } catch (TException e) { |
165 | | - LOG.warn("Failed to begin txn in channel {}, table: {}, txn: {}, msg:{}", id, targetTable, txnExecutor.getTxnId(), e.getMessage()); |
166 | | - throw e; |
167 | | - } catch (TimeoutException | InterruptedException | ExecutionException e) { |
168 | | - LOG.warn("Error occur while waiting begin txn response in channel {}, table: {}, txn: {}, msg:{}", |
| 167 | + try { |
| 168 | + // async exec begin transaction |
| 169 | + long txnId = txnExecutor.getTxnId(); |
| 170 | + if ( txnId != - 1L ) { |
| 171 | + this.txnExecutor.beginTransaction (request); |
| 172 | + LOG.info ("begin txn in channel {}, table: {}, label:{}, txn id: {}", |
| 173 | + id, targetTable, label, txnExecutor.getTxnId()); |
| 174 | + } |
| 175 | + } catch ( TException e) { |
| 176 | + LOG.warn ("Failed to begin txn in channel {}, table: {}, txn: {}, msg:{}", |
| 177 | + id, targetTable, txnExecutor.getTxnId(), e.getMessage()); |
| 178 | + throw e; |
| 179 | + } catch ( TimeoutException | InterruptedException | ExecutionException e) { |
| 180 | + LOG.warn ("Error occur while waiting begin txn response in channel {}, table: {}, txn: {}, msg:{}", |
169 | 181 | id, targetTable, txnExecutor.getTxnId(), e.getMessage()); |
170 | | - throw e; |
| 182 | + throw e; |
| 183 | + } |
| 184 | + } else { |
| 185 | + String failMsg = "current running txns on db " + db.getId() + " is " |
| 186 | + + databaseTransactionMgr.getRunningTxnNums() + ", larger than limit " + Config.max_running_txn_num_per_db; |
| 187 | + LOG.warn(failMsg); |
| 188 | + throw new BeginTransactionException(failMsg); |
171 | 189 | } |
172 | 190 | } |
173 | 191 | } |
|
0 commit comments