Skip to content

Commit 71f06ae

Browse files
morningmanimay
authored andcommitted
fix missing cluster name when get info via http (#33)
* modify thirdparty/build-thirdparty.sh to fix compilation bug of thrift. (#10) update pre-compile binaries. * fix export LD_LIBRARY_PATH in build-thirdparty.sh * fix export LD_LIBRARY_PATH in build-thirdparty.sh * fix missing cluster name bug when get load job info via http. fix compilation bugs in fe unit test
1 parent fff8e6a commit 71f06ae

30 files changed

Lines changed: 546 additions & 459 deletions

‎fe/src/com/baidu/palo/clone/CloneChecker.java‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import com.baidu.palo.catalog.Catalog;
3232
import com.baidu.palo.catalog.DataProperty;
3333
import com.baidu.palo.catalog.Database;
34+
import com.baidu.palo.catalog.Database.DbState;
3435
import com.baidu.palo.catalog.MaterializedIndex;
3536
import com.baidu.palo.catalog.MaterializedIndex.IndexState;
3637
import com.baidu.palo.catalog.OlapTable;
@@ -42,7 +43,6 @@
4243
import com.baidu.palo.catalog.Table;
4344
import com.baidu.palo.catalog.Table.TableType;
4445
import com.baidu.palo.catalog.Tablet;
45-
import com.baidu.palo.catalog.Database.DbState;
4646
import com.baidu.palo.clone.CloneJob.JobPriority;
4747
import com.baidu.palo.clone.CloneJob.JobState;
4848
import com.baidu.palo.clone.CloneJob.JobType;
@@ -238,6 +238,11 @@ private void checkTablets() {
238238
List<String> dbNames = catalog.getDbNames();
239239
for (String name : dbNames) {
240240
Database db = catalog.getDb(name);
241+
if (db == null) {
242+
LOG.warn("db does not exist. name: {}", name);
243+
continue;
244+
}
245+
241246
final String clusterName = db.getClusterName();
242247

243248
if (Strings.isNullOrEmpty(clusterName)) {
@@ -252,16 +257,13 @@ private void checkTablets() {
252257
LOG.warn("init backend infos error");
253258
continue;
254259
}
255-
final Map<Level, Set<Long>> cluserCapacityLevelToBackendIds = initBackendCapacityInfos(clusterBackendInfos);
256-
if (cluserCapacityLevelToBackendIds == null || cluserCapacityLevelToBackendIds.isEmpty()) {
260+
final Map<Level, Set<Long>> clusterCapacityLevelToBackendIds = initBackendCapacityInfos(
261+
clusterBackendInfos);
262+
if (clusterCapacityLevelToBackendIds == null || clusterCapacityLevelToBackendIds.isEmpty()) {
257263
LOG.warn("init backend capacity infos error");
258264
continue;
259265
}
260266

261-
if (db == null) {
262-
LOG.warn("db does not exist. name: {}", db.getName());
263-
continue;
264-
}
265267
long dbId = db.getId();
266268

267269
Set<String> tableNames = db.getTableNamesWithLock();
@@ -395,10 +397,10 @@ private void checkTablets() {
395397
&& !clusterDistributionLevelToBackendIds.isEmpty()) {
396398
// supplement
397399
checkSupplement(cloneTabletMap, clusterDistributionLevelToBackendIds,
398-
cluserCapacityLevelToBackendIds, clusterBackendInfos);
400+
clusterCapacityLevelToBackendIds, clusterBackendInfos);
399401
// migration
400402
checkMigration(backendToTablets, clusterDistributionLevelToBackendIds,
401-
cluserCapacityLevelToBackendIds, clusterBackendInfos);
403+
clusterCapacityLevelToBackendIds, clusterBackendInfos);
402404
} else {
403405
LOG.warn("init backend distribution infos error");
404406
}

‎fe/src/com/baidu/palo/http/BaseAction.java‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,7 @@ protected void writeCookies(BaseResponse response, HttpResponse responseObj) {
221221
}
222222

223223
public static class AuthorizationInfo {
224-
public String user;
224+
public String fullUserName;
225225
public String password;
226226
public String cluster;
227227
}
@@ -246,13 +246,14 @@ public boolean parseAuth(BaseRequest request, AuthorizationInfo authInfo) {
246246
// Note that password may contain colon, so can not simply use a
247247
// colon to split.
248248
int index = authString.indexOf(":");
249-
authInfo.user = authString.substring(0, index);
250-
final String[] elements = authInfo.user.split("@");
249+
authInfo.fullUserName = authString.substring(0, index);
250+
final String[] elements = authInfo.fullUserName.split("@");
251251
if (elements != null && elements.length < 2) {
252-
authInfo.user = ClusterNamespace.getUserFullName(SystemInfoService.DEFAULT_CLUSTER, authInfo.user);
252+
authInfo.fullUserName = ClusterNamespace.getUserFullName(SystemInfoService.DEFAULT_CLUSTER,
253+
authInfo.fullUserName);
253254
authInfo.cluster = SystemInfoService.DEFAULT_CLUSTER;
254255
} else if (elements != null && elements.length == 2) {
255-
authInfo.user = ClusterNamespace.getUserFullName(elements[1], elements[0]);
256+
authInfo.fullUserName = ClusterNamespace.getUserFullName(elements[1], elements[0]);
256257
authInfo.cluster = elements[1];
257258
}
258259
authInfo.password = authString.substring(index + 1);
@@ -272,10 +273,10 @@ private AuthorizationInfo checkAndGetUser(BaseRequest request) throws DdlExcepti
272273
if (!parseAuth(request, authInfo)) {
273274
throw new UnauthorizedException("Need auth information.");
274275
}
275-
byte[] hashedPasswd = catalog.getUserMgr().getPassword(authInfo.user);
276+
byte[] hashedPasswd = catalog.getUserMgr().getPassword(authInfo.fullUserName);
276277
if (hashedPasswd == null) {
277278
// No such user
278-
throw new DdlException("No such user(" + authInfo.user + ")");
279+
throw new DdlException("No such user(" + authInfo.fullUserName + ")");
279280
}
280281
if (!MysqlPassword.checkPlainPass(hashedPasswd, authInfo.password)) {
281282
throw new DdlException("Password error");
@@ -285,21 +286,19 @@ private AuthorizationInfo checkAndGetUser(BaseRequest request) throws DdlExcepti
285286

286287
protected void checkAdmin(BaseRequest request) throws DdlException {
287288
final AuthorizationInfo authInfo = checkAndGetUser(request);
288-
if (!catalog.getUserMgr().isAdmin(authInfo.user)) {
289+
if (!catalog.getUserMgr().isAdmin(authInfo.fullUserName)) {
289290
throw new DdlException("Administrator needed");
290291
}
291292
}
292293

293-
protected void checkReadPriv(BaseRequest request, String db) throws DdlException {
294-
final AuthorizationInfo authInfo = checkAndGetUser(request);
295-
if (!catalog.getUserMgr().checkAccess(authInfo.user, db, AccessPrivilege.READ_ONLY)) {
294+
protected void checkReadPriv(String fullUserName, String fullDbName) throws DdlException {
295+
if (!catalog.getUserMgr().checkAccess(fullUserName, fullDbName, AccessPrivilege.READ_ONLY)) {
296296
throw new DdlException("Read Privilege needed");
297297
}
298298
}
299299

300-
protected void checkWritePriv(BaseRequest request, String db) throws DdlException {
301-
final AuthorizationInfo authInfo = checkAndGetUser(request);
302-
if (!catalog.getUserMgr().checkAccess(authInfo.user, db, AccessPrivilege.READ_WRITE)) {
300+
protected void checkWritePriv(String fullUserName, String fullDbName) throws DdlException {
301+
if (!catalog.getUserMgr().checkAccess(fullUserName, fullDbName, AccessPrivilege.READ_WRITE)) {
303302
throw new DdlException("Write Privilege needed");
304303
}
305304
}

‎fe/src/com/baidu/palo/http/rest/GetLoadInfoAction.java‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,13 @@
1515

1616
package com.baidu.palo.http.rest;
1717

18+
import com.baidu.palo.cluster.ClusterNamespace;
1819
import com.baidu.palo.common.DdlException;
1920
import com.baidu.palo.http.ActionController;
2021
import com.baidu.palo.http.BaseRequest;
2122
import com.baidu.palo.http.BaseResponse;
2223
import com.baidu.palo.http.IllegalArgException;
2324
import com.baidu.palo.load.Load;
24-
2525
import com.google.common.base.Strings;
2626

2727
import io.netty.handler.codec.http.HttpMethod;
@@ -35,24 +35,30 @@ public GetLoadInfoAction(ActionController controller) {
3535
super(controller);
3636
}
3737

38-
public static void registerAction (ActionController controller)
38+
public static void registerAction(ActionController controller)
3939
throws IllegalArgException {
4040
GetLoadInfoAction action = new GetLoadInfoAction(controller);
4141
controller.registerHandler(HttpMethod.GET, "/api/{db}/_load_info", action);
4242
}
4343

4444
@Override
4545
public void execute(BaseRequest request, BaseResponse response) throws DdlException {
46-
Load.JobInfo info = new Load.JobInfo(
47-
request.getSingleParameter(DB_KEY),
48-
request.getSingleParameter(LABEL_KEY));
46+
AuthorizationInfo authInfo = getAuthorizationInfo(request);
47+
Load.JobInfo info = new Load.JobInfo(request.getSingleParameter(DB_KEY), request.getSingleParameter(LABEL_KEY),
48+
authInfo.cluster);
4949
if (Strings.isNullOrEmpty(info.dbName)) {
5050
throw new DdlException("No database selected");
5151
}
5252
if (Strings.isNullOrEmpty(info.label)) {
5353
throw new DdlException("No label selected");
5454
}
55-
checkReadPriv(request, info.dbName);
55+
if (Strings.isNullOrEmpty(info.clusterName)) {
56+
throw new DdlException("No cluster name selected");
57+
}
58+
59+
String fullDbName = ClusterNamespace.getDbFullName(info.clusterName, info.dbName);
60+
checkReadPriv(authInfo.fullUserName, fullDbName);
61+
5662
if (redirectToMaster(request, response)) {
5763
return;
5864
}

‎fe/src/com/baidu/palo/http/rest/LoadAction.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,14 +85,14 @@ public void execute(BaseRequest request, BaseResponse response) throws DdlExcept
8585
throw new DdlException("No table selected.");
8686
}
8787

88-
dbName = ClusterNamespace.getDbFullName(authInfo.cluster, dbName);
88+
String fullDbName = ClusterNamespace.getDbFullName(authInfo.cluster, dbName);
8989
String label = request.getSingleParameter(LABEL_NAME_PARAM);
9090
String subLabel = request.getSingleParameter(SUB_LABEL_NAME_PARAM);
9191
if (Strings.isNullOrEmpty(label)) {
9292
throw new DdlException("No label selected.");
9393
}
9494

95-
checkWritePriv(request, dbName);
95+
checkWritePriv(authInfo.fullUserName, fullDbName);
9696
// Try to redirect to master
9797
if (redirectToMaster(request, response)) {
9898
return;

‎fe/src/com/baidu/palo/http/rest/MultiAbort.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,15 @@
1515

1616
package com.baidu.palo.http.rest;
1717

18+
import com.baidu.palo.cluster.ClusterNamespace;
1819
import com.baidu.palo.common.DdlException;
1920
import com.baidu.palo.http.ActionController;
2021
import com.baidu.palo.http.BaseRequest;
2122
import com.baidu.palo.http.BaseResponse;
2223
import com.baidu.palo.http.IllegalArgException;
2324
import com.baidu.palo.service.ExecuteEnv;
24-
2525
import com.google.common.base.Strings;
26+
2627
import io.netty.handler.codec.http.HttpMethod;
2728

2829
public class MultiAbort extends RestBaseAction {
@@ -52,7 +53,10 @@ public void execute(BaseRequest request, BaseResponse response) throws DdlExcept
5253
if (Strings.isNullOrEmpty(label)) {
5354
throw new DdlException("No label selected");
5455
}
55-
checkWritePriv(request, db);
56+
AuthorizationInfo authInfo = getAuthorizationInfo(request);
57+
String fullDbName = ClusterNamespace.getDbFullName(authInfo.cluster, db);
58+
59+
checkWritePriv(authInfo.fullUserName, fullDbName);
5660
if (redirectToMaster(request, response)) {
5761
return;
5862
}

‎fe/src/com/baidu/palo/http/rest/MultiCommit.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,15 @@
1515

1616
package com.baidu.palo.http.rest;
1717

18+
import com.baidu.palo.cluster.ClusterNamespace;
1819
import com.baidu.palo.common.DdlException;
1920
import com.baidu.palo.http.ActionController;
2021
import com.baidu.palo.http.BaseRequest;
2122
import com.baidu.palo.http.BaseResponse;
2223
import com.baidu.palo.http.IllegalArgException;
2324
import com.baidu.palo.service.ExecuteEnv;
24-
2525
import com.google.common.base.Strings;
26+
2627
import io.netty.handler.codec.http.HttpMethod;
2728

2829
public class MultiCommit extends RestBaseAction {
@@ -52,7 +53,11 @@ public void execute(BaseRequest request, BaseResponse response) throws DdlExcept
5253
if (Strings.isNullOrEmpty(label)) {
5354
throw new DdlException("No label selected");
5455
}
55-
checkWritePriv(request, db);
56+
AuthorizationInfo authInfo = getAuthorizationInfo(request);
57+
String fullDbName = ClusterNamespace.getDbFullName(authInfo.cluster, db);
58+
59+
checkWritePriv(authInfo.fullUserName, fullDbName);
60+
5661
if (redirectToMaster(request, response)) {
5762
return;
5863
}

‎fe/src/com/baidu/palo/http/rest/MultiDesc.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,18 +15,19 @@
1515

1616
package com.baidu.palo.http.rest;
1717

18+
import java.util.List;
19+
20+
import com.baidu.palo.cluster.ClusterNamespace;
1821
import com.baidu.palo.common.DdlException;
1922
import com.baidu.palo.http.ActionController;
2023
import com.baidu.palo.http.BaseRequest;
2124
import com.baidu.palo.http.BaseResponse;
2225
import com.baidu.palo.http.IllegalArgException;
2326
import com.baidu.palo.service.ExecuteEnv;
24-
2527
import com.google.common.base.Strings;
2628
import com.google.common.collect.Lists;
27-
import io.netty.handler.codec.http.HttpMethod;
2829

29-
import java.util.List;
30+
import io.netty.handler.codec.http.HttpMethod;
3031

3132
// List all labels of one multi-load
3233
public class MultiDesc extends RestBaseAction {
@@ -56,7 +57,11 @@ public void execute(BaseRequest request, BaseResponse response) throws DdlExcept
5657
if (Strings.isNullOrEmpty(label)) {
5758
throw new DdlException("No label selected");
5859
}
59-
checkReadPriv(request, db);
60+
AuthorizationInfo authInfo = getAuthorizationInfo(request);
61+
String fullDbName = ClusterNamespace.getDbFullName(authInfo.cluster, db);
62+
63+
checkReadPriv(authInfo.fullUserName, fullDbName);
64+
6065
if (redirectToMaster(request, response)) {
6166
return;
6267
}

‎fe/src/com/baidu/palo/http/rest/MultiList.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,18 +15,19 @@
1515

1616
package com.baidu.palo.http.rest;
1717

18+
import java.util.List;
19+
20+
import com.baidu.palo.cluster.ClusterNamespace;
1821
import com.baidu.palo.common.DdlException;
1922
import com.baidu.palo.http.ActionController;
2023
import com.baidu.palo.http.BaseRequest;
2124
import com.baidu.palo.http.BaseResponse;
2225
import com.baidu.palo.http.IllegalArgException;
2326
import com.baidu.palo.service.ExecuteEnv;
24-
2527
import com.google.common.base.Strings;
2628
import com.google.common.collect.Lists;
27-
import io.netty.handler.codec.http.HttpMethod;
2829

29-
import java.util.List;
30+
import io.netty.handler.codec.http.HttpMethod;
3031

3132
// list all multi load before commit
3233
public class MultiList extends RestBaseAction {
@@ -51,7 +52,11 @@ public void execute(BaseRequest request, BaseResponse response) throws DdlExcept
5152
if (Strings.isNullOrEmpty(db)) {
5253
throw new DdlException("No database selected");
5354
}
54-
checkReadPriv(request, db);
55+
AuthorizationInfo authInfo = getAuthorizationInfo(request);
56+
String fullDbName = ClusterNamespace.getDbFullName(authInfo.cluster, db);
57+
58+
checkReadPriv(authInfo.fullUserName, fullDbName);
59+
5560
if (redirectToMaster(request, response)) {
5661
return;
5762
}

‎fe/src/com/baidu/palo/http/rest/MultiStart.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,20 @@
1515

1616
package com.baidu.palo.http.rest;
1717

18+
import java.util.Map;
19+
1820
import com.baidu.palo.analysis.LoadStmt;
21+
import com.baidu.palo.cluster.ClusterNamespace;
1922
import com.baidu.palo.common.DdlException;
2023
import com.baidu.palo.http.ActionController;
2124
import com.baidu.palo.http.BaseRequest;
2225
import com.baidu.palo.http.BaseResponse;
2326
import com.baidu.palo.http.IllegalArgException;
2427
import com.baidu.palo.service.ExecuteEnv;
25-
2628
import com.google.common.base.Strings;
2729
import com.google.common.collect.Maps;
28-
import io.netty.handler.codec.http.HttpMethod;
2930

30-
import java.util.Map;
31+
import io.netty.handler.codec.http.HttpMethod;
3132

3233
// Start multi action
3334
public class MultiStart extends RestBaseAction {
@@ -57,7 +58,10 @@ public void execute(BaseRequest request, BaseResponse response) throws DdlExcept
5758
if (Strings.isNullOrEmpty(label)) {
5859
throw new DdlException("No label selected");
5960
}
60-
checkWritePriv(request, db);
61+
AuthorizationInfo authInfo = getAuthorizationInfo(request);
62+
String fullDbName = ClusterNamespace.getDbFullName(authInfo.cluster, db);
63+
64+
checkWritePriv(authInfo.fullUserName, fullDbName);
6165

6266
if (redirectToMaster(request, response)) {
6367
return;

‎fe/src/com/baidu/palo/http/rest/MultiUnload.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,15 @@
1515

1616
package com.baidu.palo.http.rest;
1717

18+
import com.baidu.palo.cluster.ClusterNamespace;
1819
import com.baidu.palo.common.DdlException;
1920
import com.baidu.palo.http.ActionController;
2021
import com.baidu.palo.http.BaseRequest;
2122
import com.baidu.palo.http.BaseResponse;
2223
import com.baidu.palo.http.IllegalArgException;
2324
import com.baidu.palo.service.ExecuteEnv;
24-
2525
import com.google.common.base.Strings;
26+
2627
import io.netty.handler.codec.http.HttpMethod;
2728

2829
public class MultiUnload extends RestBaseAction {
@@ -57,7 +58,11 @@ public void execute(BaseRequest request, BaseResponse response) throws DdlExcept
5758
if (Strings.isNullOrEmpty(subLabel)) {
5859
throw new DdlException("No sub_label selected");
5960
}
60-
checkWritePriv(request, db);
61+
62+
AuthorizationInfo authInfo = getAuthorizationInfo(request);
63+
String fullDbName = ClusterNamespace.getDbFullName(authInfo.cluster, db);
64+
checkWritePriv(authInfo.fullUserName, fullDbName);
65+
6166
if (redirectToMaster(request, response)) {
6267
return;
6368
}

0 commit comments

Comments
 (0)