Skip to content

Commit 47aae51

Browse files
ajkannanajkannan
committed
Merge pull request #691 from ajkannan/v1beta3-read-consistency
Eventual consistency for Datastore reads in v1beta3
2 parents b4bd1b9 + 0d794ea commit 47aae51

9 files changed

Lines changed: 418 additions & 290 deletions

File tree

‎gcloud-java-datastore/pom.xml‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -42,18 +42,6 @@
4242
<artifactId>datastore-v1beta3-proto-client</artifactId>
4343
<version>0.0.1-SNAPSHOT</version>
4444
</dependency>
45-
<dependency>
46-
<groupId>com.google.apis</groupId>
47-
<artifactId>google-api-services-datastore-protobuf</artifactId>
48-
<version>v1beta2-rev1-2.1.2</version>
49-
<scope>compile</scope>
50-
<exclusions>
51-
<exclusion>
52-
<groupId>com.google.api-client</groupId>
53-
<artifactId>google-api-client</artifactId>
54-
</exclusion>
55-
</exclusions>
56-
</dependency>
5745
<dependency>
5846
<groupId>junit</groupId>
5947
<artifactId>junit</artifactId>

‎gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Datastore.java‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import com.google.gcloud.Service;
2020

21+
import java.util.Iterator;
2122
import java.util.List;
2223

2324
/**
@@ -32,7 +33,6 @@ public interface Datastore extends Service<DatastoreOptions>, DatastoreReaderWri
3233
*/
3334
Transaction newTransaction();
3435

35-
3636
/**
3737
* A callback for running with a transactional
3838
* {@link com.google.gcloud.datastore.DatastoreReaderWriter}.
@@ -45,7 +45,6 @@ interface TransactionCallable<T> {
4545
T run(DatastoreReaderWriter readerWriter) throws Exception;
4646
}
4747

48-
4948
/**
5049
* Invokes the callback's {@link Datastore.TransactionCallable#run} method with a
5150
* {@link DatastoreReaderWriter} that is associated with a new transaction.
@@ -105,4 +104,39 @@ interface TransactionCallable<T> {
105104
* Returns a new KeyFactory for this service
106105
*/
107106
KeyFactory newKeyFactory();
107+
108+
/**
109+
* Returns an {@link Entity} for the given {@link Key} or {@code null} if it doesn't exist.
110+
* {@link ReadOption}s can be specified if desired.
111+
*
112+
* @throws DatastoreException upon failure
113+
*/
114+
Entity get(Key key, ReadOption... options);
115+
116+
/**
117+
* Returns an {@link Entity} for each given {@link Key} that exists in the Datastore. The order of
118+
* the result is unspecified. Results are loaded lazily, so it is possible to get a
119+
* {@code DatastoreException} from the returned {@code Iterator}'s
120+
* {@link Iterator#hasNext hasNext} or {@link Iterator#next next} methods. {@link ReadOption}s can
121+
* be specified if desired.
122+
*
123+
* @throws DatastoreException upon failure
124+
* @see #get(Key)
125+
*/
126+
Iterator<Entity> get(Iterable<Key> keys, ReadOption... options);
127+
128+
/**
129+
* Returns a list with a value for each given key (ordered by input). {@code null} values are
130+
* returned for nonexistent keys. When possible prefer using {@link #get(Key...)} to avoid eagerly
131+
* loading the results. {@link ReadOption}s can be specified if desired.
132+
*/
133+
List<Entity> fetch(Iterable<Key> keys, ReadOption... options);
134+
135+
/**
136+
* Submits a {@link Query} and returns its result. {@link ReadOption}s can be specified if
137+
* desired.
138+
*
139+
* @throws DatastoreException upon failure
140+
*/
141+
<T> QueryResults<T> run(Query<T> query, ReadOption... options);
108142
}

‎gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
import com.google.common.collect.Maps;
2121

2222
import java.util.ArrayList;
23+
import java.util.Arrays;
24+
import java.util.Collections;
2325
import java.util.Iterator;
2426
import java.util.List;
2527
import java.util.Map;
@@ -33,13 +35,16 @@ class DatastoreHelper {
3335
private DatastoreHelper() {
3436
}
3537

36-
3738
static Key allocateId(Datastore service, IncompleteKey key) {
3839
return service.allocateId(new IncompleteKey[]{key}).get(0);
3940
}
4041

41-
static Entity get(DatastoreReader reader, Key key) {
42-
return Iterators.getNext(reader.get(new Key[]{key}), null);
42+
static Entity get(Transaction reader, Key key) {
43+
return Iterators.getNext(reader.get(new Key[] {key}), null);
44+
}
45+
46+
static Entity get(Datastore reader, Key key, ReadOption... options) {
47+
return Iterators.getNext(reader.get(Collections.singletonList(key), options), null);
4348
}
4449

4550
static Entity add(DatastoreWriter writer, FullEntity<?> entity) {
@@ -51,19 +56,30 @@ static KeyFactory newKeyFactory(DatastoreOptions options) {
5156
}
5257

5358
/**
54-
* Returns a list with a value for each given key (ordered by input).
55-
* A {@code null} would be returned for non-existing keys.
59+
* Returns a list with a value for each given key (ordered by input). {@code null} values are
60+
* returned for nonexistent keys.
5661
*/
57-
static List<Entity> fetch(DatastoreReader reader, Key... keys) {
58-
Iterator<Entity> entities = reader.get(keys);
62+
static List<Entity> fetch(Transaction reader, Key... keys) {
63+
return compileEntities(keys, reader.get(keys));
64+
}
65+
66+
/**
67+
* Returns a list with a value for each given key (ordered by input). {@code null} values are
68+
* returned for nonexistent keys.
69+
*/
70+
static List<Entity> fetch(Datastore reader, Key[] keys, ReadOption... options) {
71+
return compileEntities(keys, reader.get(Arrays.asList(keys), options));
72+
}
73+
74+
private static List<Entity> compileEntities(Key[] keys, Iterator<Entity> entities) {
5975
Map<Key, Entity> map = Maps.newHashMapWithExpectedSize(keys.length);
6076
while (entities.hasNext()) {
6177
Entity entity = entities.next();
6278
map.put(entity.key(), entity);
6379
}
6480
List<Entity> list = new ArrayList<>(keys.length);
6581
for (Key key : keys) {
66-
// this will include nulls for non-existing keys
82+
// this will include nulls for nonexistent keys
6783
list.add(map.get(key));
6884
}
6985
return list;

‎gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,14 @@
2020
import com.google.common.base.Preconditions;
2121
import com.google.common.collect.AbstractIterator;
2222
import com.google.common.collect.ImmutableList;
23+
import com.google.common.collect.Iterables;
2324
import com.google.common.collect.Sets;
25+
import com.google.datastore.v1beta3.ReadOptions.ReadConsistency;
2426
import com.google.gcloud.BaseService;
2527
import com.google.gcloud.RetryHelper;
2628
import com.google.gcloud.RetryHelper.RetryHelperException;
2729
import com.google.gcloud.RetryParams;
30+
import com.google.gcloud.datastore.ReadOption.EventualConsistency;
2831
import com.google.gcloud.spi.DatastoreRpc;
2932
import com.google.protobuf.ByteString;
3033

@@ -70,6 +73,11 @@ public <T> QueryResults<T> run(Query<T> query) {
7073
return run(null, query);
7174
}
7275

76+
@Override
77+
public <T> QueryResults<T> run(Query<T> query, ReadOption... options) {
78+
return run(toReadOptionsPb(options), query);
79+
}
80+
7381
<T> QueryResults<T> run(com.google.datastore.v1beta3.ReadOptions readOptionsPb, Query<T> query) {
7482
return new QueryResultsImpl<>(this, readOptionsPb, query);
7583
}
@@ -185,16 +193,42 @@ public Entity get(Key key) {
185193
return DatastoreHelper.get(this, key);
186194
}
187195

196+
@Override
197+
public Entity get(Key key, ReadOption... options) {
198+
return DatastoreHelper.get(this, key, options);
199+
}
200+
188201
@Override
189202
public Iterator<Entity> get(Key... keys) {
190203
return get(null, keys);
191204
}
192205

206+
@Override
207+
public Iterator<Entity> get(Iterable<Key> keys, ReadOption... options) {
208+
return get(toReadOptionsPb(options), Iterables.toArray(keys, Key.class));
209+
}
210+
211+
private static com.google.datastore.v1beta3.ReadOptions toReadOptionsPb(ReadOption... options) {
212+
com.google.datastore.v1beta3.ReadOptions readOptionsPb = null;
213+
if (options != null
214+
&& ReadOption.asImmutableMap(options).containsKey(EventualConsistency.class)) {
215+
readOptionsPb = com.google.datastore.v1beta3.ReadOptions.newBuilder()
216+
.setReadConsistency(ReadConsistency.EVENTUAL)
217+
.build();
218+
}
219+
return readOptionsPb;
220+
}
221+
193222
@Override
194223
public List<Entity> fetch(Key... keys) {
195224
return DatastoreHelper.fetch(this, keys);
196225
}
197226

227+
@Override
228+
public List<Entity> fetch(Iterable<Key> keys, ReadOption... options) {
229+
return DatastoreHelper.fetch(this, Iterables.toArray(keys, Key.class), options);
230+
}
231+
198232
Iterator<Entity> get(com.google.datastore.v1beta3.ReadOptions readOptionsPb, final Key... keys) {
199233
if (keys.length == 0) {
200234
return Collections.emptyIterator();

‎gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreReader.java‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,33 +25,32 @@
2525
public interface DatastoreReader {
2626

2727
/**
28-
* Returns an {@link Entity} for the given {@link Key} or {@code null} if does not exists.
28+
* Returns an {@link Entity} for the given {@link Key} or {@code null} if it doesn't exist.
2929
*
3030
* @throws DatastoreException upon failure
3131
*/
3232
Entity get(Key key);
3333

3434
/**
35-
* Returns an {@link Entity} for each given {@link Key} that exists in the Datastore.
36-
* The order of the result is unspecified.
37-
* Results are loaded lazily therefore it is possible to get a {@code DatastoreException}
38-
* from the returned {@code Iterator}'s {@link Iterator#hasNext hasNext} or
39-
* {@link Iterator#next next} methods.
35+
* Returns an {@link Entity} for each given {@link Key} that exists in the Datastore. The order of
36+
* the result is unspecified. Results are loaded lazily, so it is possible to get a
37+
* {@code DatastoreException} from the returned {@code Iterator}'s
38+
* {@link Iterator#hasNext hasNext} or {@link Iterator#next next} methods.
4039
*
4140
* @throws DatastoreException upon failure
4241
* @see #get(Key)
4342
*/
4443
Iterator<Entity> get(Key... key);
4544

4645
/**
47-
* Returns a list with a value for each given key (ordered by input).
48-
* A {@code null} would be returned for non-existing keys.
49-
* When possible prefer using {@link #get(Key...)} which does not eagerly loads the results.
46+
* Returns a list with a value for each given key (ordered by input). {@code null} values are
47+
* returned for nonexistent keys. When possible prefer using {@link #get(Key...)} to avoid eagerly
48+
* loading the results.
5049
*/
5150
List<Entity> fetch(Key... keys);
5251

5352
/**
54-
* Submit a {@link Query} and returns its result.
53+
* Submits a {@link Query} and returns its result.
5554
*
5655
* @throws DatastoreException upon failure
5756
*/

‎gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ReadOption.java‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Copyright 2016 Google Inc. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.gcloud.datastore;
18+
19+
import com.google.common.collect.ImmutableMap;
20+
21+
import java.io.Serializable;
22+
import java.util.Map;
23+
24+
/**
25+
* Specifies options for read operations in Datastore, namely getting/fetching entities and running
26+
* queries.
27+
*/
28+
public abstract class ReadOption implements Serializable {
29+
30+
private static final long serialVersionUID = -4406964829189800528L;
31+
32+
/**
33+
* Specifies eventual consistency for reads from Datastore. Lookups and ancestor queries using
34+
* this option permit Datastore to return stale results.
35+
*/
36+
public static final class EventualConsistency extends ReadOption {
37+
38+
private static final long serialVersionUID = -6959530217724666172L;
39+
40+
private final boolean eventualConsistency;
41+
42+
private EventualConsistency(boolean eventualConsistency) {
43+
this.eventualConsistency = eventualConsistency;
44+
}
45+
46+
public boolean isEventual() {
47+
return eventualConsistency;
48+
}
49+
}
50+
51+
private ReadOption() {}
52+
53+
/**
54+
* Returns a {@code ReadOption} that specifies eventual consistency, allowing Datastore to return
55+
* stale results from gets, fetches, and ancestor queries.
56+
*/
57+
public static EventualConsistency eventualConsistency() {
58+
return new EventualConsistency(true);
59+
}
60+
61+
static Map<Class<? extends ReadOption>, ReadOption> asImmutableMap(ReadOption... options) {
62+
ImmutableMap.Builder<Class<? extends ReadOption>, ReadOption> builder = ImmutableMap.builder();
63+
for (ReadOption option : options) {
64+
builder.put(option.getClass(), option);
65+
}
66+
return builder.build();
67+
}
68+
}

‎gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ static class ResponseImpl implements Transaction.Response {
4040

4141
@Override
4242
public List<Key> generatedKeys() {
43-
Iterator<com.google.datastore.v1beta3.MutationResult> results =
43+
Iterator<com.google.datastore.v1beta3.MutationResult> results =
4444
response.getMutationResultsList().iterator();
4545
List<Key> generated = new ArrayList<>(numAutoAllocatedIds);
4646
for (int i = 0; i < numAutoAllocatedIds; i++) {
@@ -66,7 +66,7 @@ public Entity get(Key key) {
6666
@Override
6767
public Iterator<Entity> get(Key... keys) {
6868
validateActive();
69-
com.google.datastore.v1beta3.ReadOptions.Builder readOptionsPb =
69+
com.google.datastore.v1beta3.ReadOptions.Builder readOptionsPb =
7070
com.google.datastore.v1beta3.ReadOptions.newBuilder();
7171
readOptionsPb.setTransaction(transaction);
7272
return datastore.get(readOptionsPb.build(), keys);
@@ -81,7 +81,7 @@ public List<Entity> fetch(Key... keys) {
8181
@Override
8282
public <T> QueryResults<T> run(Query<T> query) {
8383
validateActive();
84-
com.google.datastore.v1beta3.ReadOptions.Builder readOptionsPb =
84+
com.google.datastore.v1beta3.ReadOptions.Builder readOptionsPb =
8585
com.google.datastore.v1beta3.ReadOptions.newBuilder();
8686
readOptionsPb.setTransaction(transaction);
8787
return datastore.run(readOptionsPb.build(), query);
@@ -91,7 +91,7 @@ public <T> QueryResults<T> run(Query<T> query) {
9191
public Transaction.Response commit() {
9292
validateActive();
9393
List<com.google.datastore.v1beta3.Mutation> mutationsPb = toMutationPbList();
94-
com.google.datastore.v1beta3.CommitRequest.Builder requestPb =
94+
com.google.datastore.v1beta3.CommitRequest.Builder requestPb =
9595
com.google.datastore.v1beta3.CommitRequest.newBuilder();
9696
requestPb.setMode(com.google.datastore.v1beta3.CommitRequest.Mode.TRANSACTIONAL);
9797
requestPb.setTransaction(transaction);

0 commit comments

Comments
 (0)