After investigating the error I reported on this post, I have found proof that something is going wrong with azure storage performance. After around 54070 * 10 batches or 54070 * 10 * 100 rows, any insert take a disproportionate amount of time.
To reproduce it, I just inserted dummy data in the partition and monitored how much time it took to insert 10 batch.
The code is joined.
I suspect this problem is very recent, I already had so much data without any problem at all.
Here is the source code to reproduce this scaling problem (you have to wait around (54070 * 10) batches until you start having problems)
static void Main(string[] args)
{
StorageCredentials creds = new StorageCredentials("account", "pass");
CloudStorageAccount account = new CloudStorageAccount(creds, false);
var table = account.CreateCloudTableClient().GetTableReference("scaleproblem3");
table.CreateIfNotExists();
Random rand = new Random();
var random = new byte[32];
FileStream fs = File.Open("data.csv", FileMode.Append);
StreamWriter writer = new StreamWriter(fs);
int total = 0;
int count = 0;
Stopwatch watch = new Stopwatch();
watch.Start();
while (true)
{
TableBatchOperation batch = new TableBatchOperation();
for (int i = 0 ; i < 100 ; i++)
{
rand.NextBytes(random);
var rowKey = String.Join("", random.Select(b => b.ToString("00")).ToArray());
batch.Add(TableOperation.InsertOrReplace(new DynamicTableEntity("a", rowKey)));
}
table.ExecuteBatch(batch);
count++;
if (count == 10)
{
total += count;
count = 0;
var elapsedSec = (int)watch.Elapsed.TotalSeconds;
watch.Restart();
string line = total + "," + elapsedSec;
writer.WriteLine(line);
writer.Flush();
Console.WriteLine(line);
}
}
}
- Edited by NicolasDorier Saturday, January 10, 2015 7:24 PM
- Merged by Jambor yaoMicrosoft contingent staff, Moderator Monday, February 02, 2015 6:07 AM the same topic
Other recent topics
