Skip to content

Commit

Permalink
Merge pull request #1224 from busbey/pr-117-amend
Browse files Browse the repository at this point in the history 
[core] Handle merge conflicts for #117 and update it for newer bindings that reference field name prefix
  • Loading branch information
@busbey
busbey authored Nov 20, 2018
2 parents 4bff631 + 5f5ad6f commit e5f436c
Show file tree
Hide file tree
Showing 9 changed files with 47 additions and 19 deletions.
4 changes: 3 additions & 1 deletion cloudspanner/src/main/java/com/yahoo/ycsb/db/cloudspanner/CloudSpannerClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,12 +135,14 @@ private CloudSpannerProperties() {}

private static void constructStandardQueriesAndFields(Properties properties) {
String table = properties.getProperty(CoreWorkload.TABLENAME_PROPERTY, CoreWorkload.TABLENAME_PROPERTY_DEFAULT);
final String fieldprefix = properties.getProperty(CoreWorkload.FIELD_NAME_PREFIX,
CoreWorkload.FIELD_NAME_PREFIX_DEFAULT);
standardQuery = new StringBuilder()
.append("SELECT * FROM ").append(table).append(" WHERE id=@key").toString();
standardScan = new StringBuilder()
.append("SELECT * FROM ").append(table).append(" WHERE id>=@startKey LIMIT @count").toString();
for (int i = 0; i < fieldCount; i++) {
STANDARD_FIELDS.add("field" + i);
STANDARD_FIELDS.add(fieldprefix + i);
}
}

Expand Down
15 changes: 14 additions & 1 deletion core/src/main/java/com/yahoo/ycsb/workloads/CoreWorkload.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@
* digits in the record number.
* <LI><b>insertorder</b>: should records be inserted in order by key ("ordered"), or in hashed
* order ("hashed") (default: hashed)
* <LI><b>fieldnameprefix</b>: what should be a prefix for field names, the shorter may decrease the
* required storage size (default: "field")
* </ul>
*/
public class CoreWorkload extends Workload {
Expand Down Expand Up @@ -329,6 +331,16 @@ public class CoreWorkload extends Workload {
public static final String INSERTION_RETRY_INTERVAL = "core_workload_insertion_retry_interval";
public static final String INSERTION_RETRY_INTERVAL_DEFAULT = "3";

/**
* Field name prefix.
*/
public static final String FIELD_NAME_PREFIX = "fieldnameprefix";

/**
* Default value of the field name prefix.
*/
public static final String FIELD_NAME_PREFIX_DEFAULT = "field";

protected NumberGenerator keysequence;
protected DiscreteGenerator operationchooser;
protected NumberGenerator keychooser;
Expand Down Expand Up @@ -384,9 +396,10 @@ public void init(Properties p) throws WorkloadException {

fieldcount =
Long.parseLong(p.getProperty(FIELD_COUNT_PROPERTY, FIELD_COUNT_PROPERTY_DEFAULT));
final String fieldnameprefix = p.getProperty(FIELD_NAME_PREFIX, FIELD_NAME_PREFIX_DEFAULT);
fieldnames = new ArrayList<>();
for (int i = 0; i < fieldcount; i++) {
fieldnames.add("field" + i);
fieldnames.add(fieldnameprefix + i);
}
fieldlengthgenerator = CoreWorkload.getFieldLengthGenerator(p);

Expand Down
3 changes: 2 additions & 1 deletion doc/coreproperties.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ <H2>Core workload package properties</h2>
<LI><b>requestdistribution</b>: what distribution should be used to select the records to operate on - uniform, zipfian or latest (default: uniform)
<LI><b>maxscanlength</b>: for scans, what is the maximum number of records to scan (default: 1000)
<LI><b>scanlengthdistribution</b>: for scans, what distribution should be used to choose the number of records to scan, for each scan, between 1 and maxscanlength (default: uniform)
<LI><b>insertorder</b>: should records be inserted in order by key ("ordered"), or in hashed order ("hashed") (default: hashed)
<LI><b>insertorder</b>: should records be inserted in order by key ("ordered"), or in hashed order ("hashed") (default: hashed)
<LI><b>fieldnameprefix</b>: string prefix for the field name (default: “field”)
</UL>
<HR>
YCSB - Yahoo! Research - Contact [email protected].
Expand Down
8 changes: 5 additions & 3 deletions elasticsearch/src/test/java/com/yahoo/ycsb/db/ElasticsearchClientTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.yahoo.ycsb.DBException;
import com.yahoo.ycsb.Status;
import com.yahoo.ycsb.StringByteIterator;
import com.yahoo.ycsb.workloads.CoreWorkload;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
Expand All @@ -49,11 +50,12 @@ public class ElasticsearchClientTest {
private final static String MOCK_KEY0 = "0";
private final static String MOCK_KEY1 = "1";
private final static String MOCK_KEY2 = "2";
private final static String FIELD_PREFIX = CoreWorkload.FIELD_NAME_PREFIX_DEFAULT;

static {
MOCK_DATA = new HashMap<>(10);
for (int i = 1; i <= 10; i++) {
MOCK_DATA.put("field" + i, new StringByteIterator("value" + i));
MOCK_DATA.put(FIELD_PREFIX + i, new StringByteIterator("value" + i));
}
}

Expand Down Expand Up @@ -120,7 +122,7 @@ public void testUpdate() {
HashMap<String, ByteIterator> newValues = new HashMap<>(10);

for (i = 1; i <= 10; i++) {
newValues.put("field" + i, new StringByteIterator("newvalue" + i));
newValues.put(FIELD_PREFIX + i, new StringByteIterator("newvalue" + i));
}

Status result = instance.update(MOCK_TABLE, MOCK_KEY1, newValues);
Expand All @@ -131,7 +133,7 @@ public void testUpdate() {
instance.read(MOCK_TABLE, MOCK_KEY1, MOCK_DATA.keySet(), resultParam);

for (i = 1; i <= 10; i++) {
assertEquals("newvalue" + i, resultParam.get("field" + i).toString());
assertEquals("newvalue" + i, resultParam.get(FIELD_PREFIX + i).toString());
}
}

Expand Down
8 changes: 5 additions & 3 deletions ...ticsearch5/src/test/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchIntegTestBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import com.yahoo.ycsb.DBException;
import com.yahoo.ycsb.Status;
import com.yahoo.ycsb.StringByteIterator;
import com.yahoo.ycsb.workloads.CoreWorkload;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
Expand All @@ -41,11 +42,12 @@ public abstract class ElasticsearchIntegTestBase {

private final static HashMap<String, ByteIterator> MOCK_DATA;
private final static String MOCK_TABLE = "MOCK_TABLE";
private final static String FIELD_PREFIX = CoreWorkload.FIELD_NAME_PREFIX_DEFAULT;

static {
MOCK_DATA = new HashMap<>(10);
for (int i = 1; i <= 10; i++) {
MOCK_DATA.put("field" + i, new StringByteIterator("value" + i));
MOCK_DATA.put(FIELD_PREFIX + i, new StringByteIterator("value" + i));
}
}

Expand Down Expand Up @@ -101,7 +103,7 @@ public void testUpdate() {
final HashMap<String, ByteIterator> newValues = new HashMap<>(10);

for (int i = 1; i <= 10; i++) {
newValues.put("field" + i, new StringByteIterator("newvalue" + i));
newValues.put(FIELD_PREFIX + i, new StringByteIterator("newvalue" + i));
}

final Status updateResult = db.update(MOCK_TABLE, "1", newValues);
Expand All @@ -113,7 +115,7 @@ public void testUpdate() {
assertEquals(Status.OK, readResult);

for (int i = 1; i <= 10; i++) {
assertEquals("newvalue" + i, resultParam.get("field" + i).toString());
assertEquals("newvalue" + i, resultParam.get(FIELD_PREFIX + i).toString());
}

}
Expand Down
4 changes: 3 additions & 1 deletion kudu/src/main/java/com/yahoo/ycsb/db/KuduYCSBClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,8 @@ private void setupTable() throws DBException {
int blockSize = getIntFromProp(prop, BLOCK_SIZE_OPT, BLOCK_SIZE_DEFAULT);
int fieldCount = getIntFromProp(prop, CoreWorkload.FIELD_COUNT_PROPERTY,
Integer.parseInt(CoreWorkload.FIELD_COUNT_PROPERTY_DEFAULT));
final String fieldprefix = prop.getProperty(CoreWorkload.FIELD_NAME_PREFIX,
CoreWorkload.FIELD_NAME_PREFIX_DEFAULT);

List<ColumnSchema> columns = new ArrayList<ColumnSchema>(fieldCount + 1);

Expand All @@ -176,7 +178,7 @@ private void setupTable() throws DBException {
columns.add(keyColumn);
COLUMN_NAMES.add(KEY);
for (int i = 0; i < fieldCount; i++) {
String name = "field" + i;
String name = fieldprefix + i;
COLUMN_NAMES.add(name);
columns.add(new ColumnSchema.ColumnSchemaBuilder(name, STRING)
.desiredBlockSize(blockSize)
Expand Down
8 changes: 5 additions & 3 deletions rocksdb/src/test/java/com/yahoo/ycsb/db/rocksdb/RocksDBClientTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.yahoo.ycsb.ByteIterator;
import com.yahoo.ycsb.Status;
import com.yahoo.ycsb.StringByteIterator;
import com.yahoo.ycsb.workloads.CoreWorkload;
import org.junit.*;
import org.junit.rules.TemporaryFolder;

Expand All @@ -38,12 +39,13 @@ public class RocksDBClientTest {
private static final String MOCK_KEY2 = "2";
private static final String MOCK_KEY3 = "3";
private static final int NUM_RECORDS = 10;
private static final String FIELD_PREFIX = CoreWorkload.FIELD_NAME_PREFIX_DEFAULT;

private static final Map<String, ByteIterator> MOCK_DATA;
static {
MOCK_DATA = new HashMap<>(NUM_RECORDS);
for (int i = 0; i < NUM_RECORDS; i++) {
MOCK_DATA.put("field" + i, new StringByteIterator("value" + i));
MOCK_DATA.put(FIELD_PREFIX + i, new StringByteIterator("value" + i));
}
}

Expand Down Expand Up @@ -93,7 +95,7 @@ public void insertUpdateAndRead() throws Exception {
assertEquals(Status.OK, insertResult);

for (int i = 0; i < NUM_RECORDS; i++) {
newValues.put("field" + i, new StringByteIterator("newvalue" + i));
newValues.put(FIELD_PREFIX + i, new StringByteIterator("newvalue" + i));
}

final Status result = instance.update(MOCK_TABLE, MOCK_KEY2, newValues);
Expand All @@ -104,7 +106,7 @@ public void insertUpdateAndRead() throws Exception {
instance.read(MOCK_TABLE, MOCK_KEY2, MOCK_DATA.keySet(), resultParam);

for (int i = 0; i < NUM_RECORDS; i++) {
assertEquals("newvalue" + i, resultParam.get("field" + i).toString());
assertEquals("newvalue" + i, resultParam.get(FIELD_PREFIX + i).toString());
}
}

Expand Down
8 changes: 5 additions & 3 deletions solr/src/test/java/com/yahoo/ycsb/db/solr/SolrClientBaseTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.yahoo.ycsb.DB;
import com.yahoo.ycsb.Status;
import com.yahoo.ycsb.StringByteIterator;
import com.yahoo.ycsb.workloads.CoreWorkload;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.common.util.NamedList;
Expand All @@ -47,11 +48,12 @@ public abstract class SolrClientBaseTest {
private final static String MOCK_KEY0 = "0";
private final static String MOCK_KEY1 = "1";
private final static int NUM_RECORDS = 10;
private final static String FIELD_PREFIX = CoreWorkload.FIELD_NAME_PREFIX_DEFAULT;

static {
MOCK_DATA = new HashMap<>(NUM_RECORDS);
for (int i = 0; i < NUM_RECORDS; i++) {
MOCK_DATA.put("field" + i, new StringByteIterator("value" + i));
MOCK_DATA.put(FIELD_PREFIX + i, new StringByteIterator("value" + i));
}
}

Expand Down Expand Up @@ -117,7 +119,7 @@ public void testUpdate() throws Exception {
HashMap<String, ByteIterator> newValues = new HashMap<>(NUM_RECORDS);

for (int i = 0; i < NUM_RECORDS; i++) {
newValues.put("field" + i, new StringByteIterator("newvalue" + i));
newValues.put(FIELD_PREFIX + i, new StringByteIterator("newvalue" + i));
}

Status result = instance.update(MOCK_TABLE, MOCK_KEY1, newValues);
Expand All @@ -128,7 +130,7 @@ public void testUpdate() throws Exception {
instance.read(MOCK_TABLE, MOCK_KEY1, MOCK_DATA.keySet(), resultParam);

for (int i = 0; i < NUM_RECORDS; i++) {
assertEquals("newvalue" + i, resultParam.get("field" + i).toString());
assertEquals("newvalue" + i, resultParam.get(FIELD_PREFIX + i).toString());
}
}

Expand Down
8 changes: 5 additions & 3 deletions solr6/src/test/java/com/yahoo/ycsb/db/solr6/SolrClientBaseTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.yahoo.ycsb.DB;
import com.yahoo.ycsb.Status;
import com.yahoo.ycsb.StringByteIterator;
import com.yahoo.ycsb.workloads.CoreWorkload;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.common.util.NamedList;
Expand All @@ -47,11 +48,12 @@ public abstract class SolrClientBaseTest {
private final static String MOCK_KEY0 = "0";
private final static String MOCK_KEY1 = "1";
private final static int NUM_RECORDS = 10;
private final static String FIELD_PREFIX = CoreWorkload.FIELD_NAME_PREFIX_DEFAULT;

static {
MOCK_DATA = new HashMap<>(NUM_RECORDS);
for (int i = 0; i < NUM_RECORDS; i++) {
MOCK_DATA.put("field" + i, new StringByteIterator("value" + i));
MOCK_DATA.put(FIELD_PREFIX + i, new StringByteIterator("value" + i));
}
}

Expand Down Expand Up @@ -117,7 +119,7 @@ public void testUpdate() throws Exception {
HashMap<String, ByteIterator> newValues = new HashMap<>(NUM_RECORDS);

for (int i = 0; i < NUM_RECORDS; i++) {
newValues.put("field" + i, new StringByteIterator("newvalue" + i));
newValues.put(FIELD_PREFIX + i, new StringByteIterator("newvalue" + i));
}

Status result = instance.update(MOCK_TABLE, MOCK_KEY1, newValues);
Expand All @@ -128,7 +130,7 @@ public void testUpdate() throws Exception {
instance.read(MOCK_TABLE, MOCK_KEY1, MOCK_DATA.keySet(), resultParam);

for (int i = 0; i < NUM_RECORDS; i++) {
assertEquals("newvalue" + i, resultParam.get("field" + i).toString());
assertEquals("newvalue" + i, resultParam.get(FIELD_PREFIX + i).toString());
}
}

Expand Down

0 comments on commit e5f436c

Please sign in to comment.