Skip to content

Commit d45bc2c

Browse files
codemzscodemzs
committed
Merge master.
1 parent 8c80530 commit d45bc2c

4 files changed

Lines changed: 62 additions & 60 deletions

File tree

‎src/Microsoft.ML.Data/Transforms/KeyToVectorTransform.cs‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -313,50 +313,50 @@ private void AddMetadata(int i, ColumnMetadataInfo colMetaInfo)
313313
{
314314
if (typeNames != null)
315315
{
316-
MetadataUtils.MetadataGetter<VBuffer<DvText>> getter = (int col, ref VBuffer<DvText> dst) =>
316+
MetadataUtils.MetadataGetter<VBuffer<ReadOnlyMemory<char>>> getter = (int col, ref VBuffer<ReadOnlyMemory<char>> dst) =>
317317
{
318318
InputSchema.GetMetadata(MetadataUtils.Kinds.KeyValues, srcCol, ref dst);
319319
};
320-
var info = new MetadataInfo<VBuffer<DvText>>(typeNames, getter);
320+
var info = new MetadataInfo<VBuffer<ReadOnlyMemory<char>>>(typeNames, getter);
321321
colMetaInfo.Add(MetadataUtils.Kinds.SlotNames, info);
322322
}
323323
}
324324
else
325325
{
326326
if (typeNames != null && _types[i].IsKnownSizeVector)
327327
{
328-
MetadataUtils.MetadataGetter<VBuffer<DvText>> getter = (int col, ref VBuffer<DvText> dst) =>
328+
MetadataUtils.MetadataGetter<VBuffer<ReadOnlyMemory<char>>> getter = (int col, ref VBuffer<ReadOnlyMemory<char>> dst) =>
329329
{
330330
GetSlotNames(i, ref dst);
331331
};
332-
var info = new MetadataInfo<VBuffer<DvText>>(new VectorType(TextType.Instance, _types[i]), getter);
332+
var info = new MetadataInfo<VBuffer<ReadOnlyMemory<char>>>(new VectorType(TextType.Instance, _types[i]), getter);
333333
colMetaInfo.Add(MetadataUtils.Kinds.SlotNames, info);
334334
}
335335
}
336336

337337
if (!_parent._columns[i].Bag && srcType.ValueCount > 0)
338338
{
339-
MetadataUtils.MetadataGetter<VBuffer<DvInt4>> getter = (int col, ref VBuffer<DvInt4> dst) =>
339+
MetadataUtils.MetadataGetter<VBuffer<int>> getter = (int col, ref VBuffer<int> dst) =>
340340
{
341341
GetCategoricalSlotRanges(i, ref dst);
342342
};
343-
var info = new MetadataInfo<VBuffer<DvInt4>>(MetadataUtils.GetCategoricalType(_infos[i].TypeSrc.ValueCount), getter);
343+
var info = new MetadataInfo<VBuffer<int>>(MetadataUtils.GetCategoricalType(_infos[i].TypeSrc.ValueCount), getter);
344344
colMetaInfo.Add(MetadataUtils.Kinds.CategoricalSlotRanges, info);
345345
}
346346

347347
if (!_parent._columns[i].Bag || srcType.ValueCount == 1)
348348
{
349-
MetadataUtils.MetadataGetter<DvBool> getter = (int col, ref DvBool dst) =>
349+
MetadataUtils.MetadataGetter<bool> getter = (int col, ref bool dst) =>
350350
{
351351
dst = true;
352352
};
353-
var info = new MetadataInfo<DvBool>(BoolType.Instance, getter);
353+
var info = new MetadataInfo<bool>(BoolType.Instance, getter);
354354
colMetaInfo.Add(MetadataUtils.Kinds.IsNormalized, info);
355355
}
356356
}
357357

358358
// Combines source key names and slot names to produce final slot names.
359-
private void GetSlotNames(int iinfo, ref VBuffer<DvText> dst)
359+
private void GetSlotNames(int iinfo, ref VBuffer<ReadOnlyMemory<char>> dst)
360360
{
361361
Host.Assert(0 <= iinfo && iinfo < _infos.Length);
362362
Host.Assert(_types[iinfo].IsKnownSizeVector);
@@ -367,7 +367,7 @@ private void GetSlotNames(int iinfo, ref VBuffer<DvText> dst)
367367
Host.Assert(typeSrc.VectorSize > 1);
368368

369369
// Get the source slot names, defaulting to empty text.
370-
var namesSlotSrc = default(VBuffer<DvText>);
370+
var namesSlotSrc = default(VBuffer<ReadOnlyMemory<char>>);
371371
InputSchema.TryGetColumnIndex(_infos[iinfo].Source, out int srcCol);
372372
Host.Assert(srcCol >= 0);
373373
var typeSlotSrc = InputSchema.GetMetadataTypeOrNull(MetadataUtils.Kinds.SlotNames, srcCol);
@@ -377,31 +377,31 @@ private void GetSlotNames(int iinfo, ref VBuffer<DvText> dst)
377377
Host.Check(namesSlotSrc.Length == typeSrc.VectorSize);
378378
}
379379
else
380-
namesSlotSrc = VBufferUtils.CreateEmpty<DvText>(typeSrc.VectorSize);
380+
namesSlotSrc = VBufferUtils.CreateEmpty<ReadOnlyMemory<char>>(typeSrc.VectorSize);
381381

382382
int keyCount = typeSrc.ItemType.ItemType.KeyCount;
383383
int slotLim = _types[iinfo].VectorSize;
384384
Host.Assert(slotLim == (long)typeSrc.VectorSize * keyCount);
385385

386386
// Get the source key names, in an array (since we will use them multiple times).
387-
var namesKeySrc = default(VBuffer<DvText>);
387+
var namesKeySrc = default(VBuffer<ReadOnlyMemory<char>>);
388388
InputSchema.GetMetadata(MetadataUtils.Kinds.KeyValues, srcCol, ref namesKeySrc);
389389
Host.Check(namesKeySrc.Length == keyCount);
390-
var keys = new DvText[keyCount];
390+
var keys = new ReadOnlyMemory<char>[keyCount];
391391
namesKeySrc.CopyTo(keys);
392392

393393
var values = dst.Values;
394394
if (Utils.Size(values) < slotLim)
395-
values = new DvText[slotLim];
395+
values = new ReadOnlyMemory<char>[slotLim];
396396

397397
var sb = new StringBuilder();
398398
int slot = 0;
399399
foreach (var kvpSlot in namesSlotSrc.Items(all: true))
400400
{
401401
Contracts.Assert(slot == (long)kvpSlot.Key * keyCount);
402402
sb.Clear();
403-
if (kvpSlot.Value.HasChars)
404-
kvpSlot.Value.AddToStringBuilder(sb);
403+
if (!kvpSlot.Value.IsEmpty)
404+
ReadOnlyMemoryUtils.AddToStringBuilder(kvpSlot.Value, sb);
405405
else
406406
sb.Append('[').Append(kvpSlot.Key).Append(']');
407407
sb.Append('.');
@@ -410,24 +410,24 @@ private void GetSlotNames(int iinfo, ref VBuffer<DvText> dst)
410410
foreach (var key in keys)
411411
{
412412
sb.Length = len;
413-
key.AddToStringBuilder(sb);
414-
values[slot++] = new DvText(sb.ToString());
413+
ReadOnlyMemoryUtils.AddToStringBuilder(key, sb);
414+
values[slot++] = sb.ToString().AsMemory();
415415
}
416416
}
417417
Host.Assert(slot == slotLim);
418418

419-
dst = new VBuffer<DvText>(slotLim, values, dst.Indices);
419+
dst = new VBuffer<ReadOnlyMemory<char>>(slotLim, values, dst.Indices);
420420
}
421421

422-
private void GetCategoricalSlotRanges(int iinfo, ref VBuffer<DvInt4> dst)
422+
private void GetCategoricalSlotRanges(int iinfo, ref VBuffer<int> dst)
423423
{
424424
Host.Assert(0 <= iinfo && iinfo < _infos.Length);
425425

426426
var info = _infos[iinfo];
427427

428428
Host.Assert(info.TypeSrc.ValueCount > 0);
429429

430-
DvInt4[] ranges = new DvInt4[info.TypeSrc.ValueCount * 2];
430+
int[] ranges = new int[info.TypeSrc.ValueCount * 2];
431431
int size = info.TypeSrc.ItemType.KeyCount;
432432

433433
ranges[0] = 0;
@@ -438,7 +438,7 @@ private void GetCategoricalSlotRanges(int iinfo, ref VBuffer<DvInt4> dst)
438438
ranges[i + 1] = ranges[i] + size - 1;
439439
}
440440

441-
dst = new VBuffer<DvInt4>(ranges.Length, ranges);
441+
dst = new VBuffer<int>(ranges.Length, ranges);
442442
}
443443

444444
protected override Delegate MakeGetter(IRow input, int iinfo, out Action disposer)
@@ -834,7 +834,7 @@ public OutVectorColumn(Vector<Key<TKey>> input, bool bag)
834834
Input = input;
835835
Bag = bag;
836836
}
837-
837+
838838
public OutVectorColumn(VarVector<Key<TKey>> input)
839839
: base(Reconciler.Inst, input)
840840
{

‎src/Microsoft.ML.Transforms/KeyToBinaryVectorTransform.cs‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -246,46 +246,46 @@ private void AddMetadata(int i, ColumnMetadataInfo colMetaInfo)
246246
{
247247
if (typeNames != null)
248248
{
249-
MetadataUtils.MetadataGetter<VBuffer<DvText>> getter = (int col, ref VBuffer<DvText> dst) =>
249+
MetadataUtils.MetadataGetter<VBuffer<ReadOnlyMemory<char>>> getter = (int col, ref VBuffer<ReadOnlyMemory<char>> dst) =>
250250
{
251251
GenerateBitSlotName(i, ref dst);
252252
};
253-
var info = new MetadataInfo<VBuffer<DvText>>(new VectorType(TextType.Instance, _types[i]), getter);
253+
var info = new MetadataInfo<VBuffer<ReadOnlyMemory<char>>>(new VectorType(TextType.Instance, _types[i]), getter);
254254
colMetaInfo.Add(MetadataUtils.Kinds.SlotNames, info);
255255
}
256-
MetadataUtils.MetadataGetter<DvBool> normalizeGetter = (int col, ref DvBool dst) =>
256+
MetadataUtils.MetadataGetter<bool> normalizeGetter = (int col, ref bool dst) =>
257257
{
258258
dst = true;
259259
};
260-
var normalizeInfo = new MetadataInfo<DvBool>(BoolType.Instance, normalizeGetter);
260+
var normalizeInfo = new MetadataInfo<bool>(BoolType.Instance, normalizeGetter);
261261
colMetaInfo.Add(MetadataUtils.Kinds.IsNormalized, normalizeInfo);
262262
}
263263
else
264264
{
265265
if (typeNames != null && _types[i].IsKnownSizeVector)
266266
{
267-
MetadataUtils.MetadataGetter<VBuffer<DvText>> getter = (int col, ref VBuffer<DvText> dst) =>
267+
MetadataUtils.MetadataGetter<VBuffer<ReadOnlyMemory<char>>> getter = (int col, ref VBuffer<ReadOnlyMemory<char>> dst) =>
268268
{
269269
GetSlotNames(i, ref dst);
270270
};
271-
var info = new MetadataInfo<VBuffer<DvText>>(new VectorType(TextType.Instance, _types[i]), getter);
271+
var info = new MetadataInfo<VBuffer<ReadOnlyMemory<char>>>(new VectorType(TextType.Instance, _types[i]), getter);
272272
colMetaInfo.Add(MetadataUtils.Kinds.SlotNames, info);
273273
}
274274
}
275275
}
276276

277-
private void GenerateBitSlotName(int iinfo, ref VBuffer<DvText> dst)
277+
private void GenerateBitSlotName(int iinfo, ref VBuffer<ReadOnlyMemory<char>> dst)
278278
{
279279
const string slotNamePrefix = "Bit";
280-
var bldr = new BufferBuilder<DvText>(TextCombiner.Instance);
280+
var bldr = new BufferBuilder<ReadOnlyMemory<char>>(TextCombiner.Instance);
281281
bldr.Reset(_bitsPerKey[iinfo], true);
282282
for (int i = 0; i < _bitsPerKey[iinfo]; i++)
283-
bldr.AddFeature(i, new DvText(slotNamePrefix + (_bitsPerKey[iinfo] - i - 1)));
283+
bldr.AddFeature(i, (slotNamePrefix + (_bitsPerKey[iinfo] - i - 1)).AsMemory());
284284

285285
bldr.GetResult(ref dst);
286286
}
287287

288-
private void GetSlotNames(int iinfo, ref VBuffer<DvText> dst)
288+
private void GetSlotNames(int iinfo, ref VBuffer<ReadOnlyMemory<char>> dst)
289289
{
290290
Host.Assert(0 <= iinfo && iinfo < _infos.Length);
291291
Host.Assert(_types[iinfo].IsKnownSizeVector);
@@ -295,7 +295,7 @@ private void GetSlotNames(int iinfo, ref VBuffer<DvText> dst)
295295
Host.Assert(typeSrc.VectorSize > 1);
296296

297297
// Get the source slot names, defaulting to empty text.
298-
var namesSlotSrc = default(VBuffer<DvText>);
298+
var namesSlotSrc = default(VBuffer<ReadOnlyMemory<char>>);
299299
InputSchema.TryGetColumnIndex(_infos[iinfo].Source, out int srcCol);
300300
Host.Assert(srcCol >= 0);
301301
var typeSlotSrc = InputSchema.GetMetadataTypeOrNull(MetadataUtils.Kinds.SlotNames, srcCol);
@@ -305,25 +305,25 @@ private void GetSlotNames(int iinfo, ref VBuffer<DvText> dst)
305305
Host.Check(namesSlotSrc.Length == typeSrc.VectorSize);
306306
}
307307
else
308-
namesSlotSrc = VBufferUtils.CreateEmpty<DvText>(typeSrc.VectorSize);
308+
namesSlotSrc = VBufferUtils.CreateEmpty<ReadOnlyMemory<char>>(typeSrc.VectorSize);
309309

310310
int slotLim = _types[iinfo].VectorSize;
311311
Host.Assert(slotLim == (long)typeSrc.VectorSize * _bitsPerKey[iinfo]);
312312

313313
var values = dst.Values;
314314
if (Utils.Size(values) < slotLim)
315-
values = new DvText[slotLim];
315+
values = new ReadOnlyMemory<char>[slotLim];
316316

317317
var sb = new StringBuilder();
318318
int slot = 0;
319-
VBuffer<DvText> bits = default;
319+
VBuffer<ReadOnlyMemory<char>> bits = default;
320320
GenerateBitSlotName(iinfo, ref bits);
321321
foreach (var kvpSlot in namesSlotSrc.Items(all: true))
322322
{
323323
Contracts.Assert(slot == (long)kvpSlot.Key * _bitsPerKey[iinfo]);
324324
sb.Clear();
325-
if (kvpSlot.Value.HasChars)
326-
kvpSlot.Value.AddToStringBuilder(sb);
325+
if (!kvpSlot.Value.IsEmpty)
326+
ReadOnlyMemoryUtils.AddToStringBuilder(kvpSlot.Value, sb);
327327
else
328328
sb.Append('[').Append(kvpSlot.Key).Append(']');
329329
sb.Append('.');
@@ -332,13 +332,13 @@ private void GetSlotNames(int iinfo, ref VBuffer<DvText> dst)
332332
foreach (var key in bits.Values)
333333
{
334334
sb.Length = len;
335-
key.AddToStringBuilder(sb);
336-
values[slot++] = new DvText(sb.ToString());
335+
ReadOnlyMemoryUtils.AddToStringBuilder(key, sb);
336+
values[slot++] = sb.ToString().AsMemory();
337337
}
338338
}
339339
Host.Assert(slot == slotLim);
340340

341-
dst = new VBuffer<DvText>(slotLim, values, dst.Indices);
341+
dst = new VBuffer<ReadOnlyMemory<char>>(slotLim, values, dst.Indices);
342342
}
343343

344344
protected override Delegate MakeGetter(IRow input, int iinfo, out Action disposer)

‎test/Microsoft.ML.Tests/Transformers/KeyToBinaryVectorEstimatorTest.cs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using Microsoft.ML.Runtime.Model;
99
using Microsoft.ML.Runtime.RunTests;
1010
using Microsoft.ML.Runtime.Tools;
11+
using System;
1112
using System.IO;
1213
using System.Linq;
1314
using Xunit;
@@ -124,8 +125,8 @@ private void ValidateMetadata(IDataView result)
124125
Assert.True(result.Schema.TryGetColumnIndex("CatD", out int colD));
125126
var types = result.Schema.GetMetadataTypes(colA);
126127
Assert.Equal(types.Select(x => x.Key), new string[1] { MetadataUtils.Kinds.SlotNames });
127-
VBuffer<DvText> slots = default;
128-
DvBool normalized = default;
128+
VBuffer<ReadOnlyMemory<char>> slots = default;
129+
bool normalized = default;
129130
result.Schema.GetMetadata(MetadataUtils.Kinds.SlotNames, colA, ref slots);
130131
Assert.True(slots.Length == 6);
131132
Assert.Equal(slots.Values.Select(x => x.ToString()), new string[6] { "[0].Bit2", "[0].Bit1", "[0].Bit0", "[1].Bit2", "[1].Bit1", "[1].Bit0" });
@@ -136,15 +137,15 @@ private void ValidateMetadata(IDataView result)
136137
Assert.True(slots.Length == 2);
137138
Assert.Equal(slots.Items().Select(x => x.Value.ToString()), new string[2] { "Bit1", "Bit0" });
138139
result.Schema.GetMetadata(MetadataUtils.Kinds.IsNormalized, colB, ref normalized);
139-
Assert.True(normalized.IsTrue);
140+
Assert.True(normalized);
140141

141142
types = result.Schema.GetMetadataTypes(colC);
142143
Assert.Equal(types.Select(x => x.Key), new string[0]);
143144

144145
types = result.Schema.GetMetadataTypes(colD);
145146
Assert.Equal(types.Select(x => x.Key), new string[1] { MetadataUtils.Kinds.IsNormalized });
146147
result.Schema.GetMetadata(MetadataUtils.Kinds.IsNormalized, colD, ref normalized);
147-
Assert.True(normalized.IsTrue);
148+
Assert.True(normalized);
148149
}
149150

150151
[Fact]

0 commit comments

Comments
 (0)