using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; //using log4net; using MongoDB.Driver; using System.IO; using Newtonsoft.Json.Linq; using MongoDB.Bson.Serialization.Attributes; using MongoDB.Bson; namespace AIAHighlight.DBSave { [BsonIgnoreExtraElements] public class BodyRegionImage { public ObjectId _id { get; set; } public string imageName; public int bodyViewIndex; public int layerNumber; public int bodyRegionId; public int zoom; public List terms; public BodyRegionImage() { this.terms = new List(); } } public class Term { public int TermNumber; public List SystemNumbers; public List PixelLocations; public int ActualTermNumber; public Term(int termNumber, int systemNumber, int pixelLocation) { this.SystemNumbers = new List(); this.PixelLocations = new List(); this.TermNumber = termNumber; this.SystemNumbers.Add(systemNumber); this.PixelLocations.Add(pixelLocation); } } public class Program { //public JObject data { get; set; } // public Dictionary termList; internal void saveImageData(JObject data) { // DateTime datetime = new DateTime(); Console.WriteLine("AT "+ DateTime.Now.ToString("yyyy-MM-ddThh:mm:sszzz")+"..In saveImageData. for imageName: "+ data["imageName"].ToString()+ ",bodyViewIndex= "+ data["bodyViewIndex"]+ ",layerNumber: "+ data["layerNumber"]+ ",bodyRegionId= "+ data["bodyRegion"]); try { BodyRegionImage bodyRegionImage = new BodyRegionImage(); var client = new MongoClient(); var db = client.GetDatabase("AIAImageData_25"); var col = db.GetCollection("DAImages"); JObject imageData = data; string bodyViewIndex = data["bodyViewIndex"].ToString(); string termFilePath = @"C:\ADAM-AWS-FTP\ADAM-AWS\AIAHTML5\bodyViewsData\body-views\" + bodyViewIndex + "\\da_dat_body_system_term_" + bodyViewIndex + ".json"; //string termFilePath = @"D:\100-PROJECTS\008-AIAHighlight.FileSave\dist\assets\data\json\da\body-views\" + bodyViewIndex + "\\da_dat_body_system_term_" + bodyViewIndex + ".json"; string systemTermMapping; using (StreamReader reader = new StreamReader(termFilePath)) // using (StreamReader reader = new StreamReader(path+fileName)) { systemTermMapping = reader.ReadToEnd(); } JObject systemTermMappingObject = JObject.Parse(systemTermMapping); //Console.WriteLine("AT " + DateTime.Now.ToString("yyyy-MM-ddThh:mm:sszzz") + ".systemTermMappingObject created."); dynamic systemTerms = systemTermMappingObject["BodySystem"]["BodySystemTerm"]; dynamic imageDataArray = data["data"]; Dictionary termList = new Dictionary(); Console.WriteLine("AT " + DateTime.Now.ToString("yyyy-MM-ddThh:mm:sszzz") + "..before itrating imageDataArray. for imageName: " + data["imageName"].ToString() + ",bodyViewIndex= " + data["bodyViewIndex"] + ",layerNumber: " + data["layerNumber"] + ",bodyRegionId= " + data["bodyRegion"]); for (int i = 0; i < imageDataArray.Count; i += 4) { if(imageDataArray[i] == 0 && imageDataArray[i+1] == 0 && imageDataArray[i+2] == 0) { continue; } var RED = Convert.ToInt32(imageDataArray[i]) - 9; var GREEN = Convert.ToInt32(imageDataArray[i + 1]) - 9; var BLUE = Convert.ToInt32(imageDataArray[i + 2]) - 9; if (RED == -9 && GREEN == -9 && BLUE == -9) continue; var Red = ""; var Green = ""; var Blue = ""; string zero = "0"; if ((RED).ToString().Length != 2) { Red = zero + RED.ToString(); } else { Red = RED.ToString(); } if ((GREEN).ToString().Length != 2) { Green = zero + GREEN.ToString(); } else { Green = GREEN.ToString(); } if ((BLUE).ToString().Length != 2) { Blue = zero + BLUE.ToString(); } else { Blue = BLUE.ToString(); } string Icolor = (Red + Green + Blue); foreach (dynamic systemTerm in systemTerms) { int iColor = Convert.ToInt32(Icolor); int termNumber = Convert.ToInt32(systemTerm._TermNumber.ToString()); int systemNumber = Convert.ToInt32(systemTerm._SystemNumber.ToString()); if (systemTerm._TermNumber.ToString() == Icolor) { this.AddToTermList(termNumber, systemNumber, i, termList); //this.AddToTermList(termNumber, systemNumber, i, termList); this.AddToTermList(termNumber, systemNumber, i + 1, termList); this.AddToTermList(termNumber, systemNumber, i + 2, termList); break; } } } Console.WriteLine("AT " + DateTime.Now.ToString("yyyy-MM-ddThh:mm:sszzz") + "..after craeting termList.. for imageName: " + data["imageName"].ToString() + ",bodyViewIndex= " + data["bodyViewIndex"] + ",layerNumber: " + data["layerNumber"] + ",bodyRegionId= " + data["bodyRegion"]); bodyRegionImage.imageName = data["imageName"].ToString(); bodyRegionImage.bodyViewIndex = Convert.ToInt32(data["bodyViewIndex"]); bodyRegionImage.layerNumber = Convert.ToInt32(data["layerNumber"]); bodyRegionImage.bodyRegionId = Convert.ToInt32(data["bodyRegion"]); bodyRegionImage.zoom = Convert.ToInt32(data["zoom"]); foreach (int key in termList.Keys) { bodyRegionImage.terms.Add(termList[key]); } Console.WriteLine("AT " + DateTime.Now.ToString("yyyy-MM-ddThh:mm:sszzz") + "...inserting for imageName: " + data["imageName"].ToString() + ",bodyViewIndex= " + data["bodyViewIndex"] + ",layerNumber: " + data["layerNumber"] + ",bodyRegionId= " + data["bodyRegion"]); col.InsertOne(bodyRegionImage); Console.WriteLine("AT " + DateTime.Now.ToString("yyyy-MM-ddThh:mm:sszzz") + "..inserted for imageName: " + data["imageName"].ToString() + ",bodyViewIndex= " + data["bodyViewIndex"] + ",layerNumber: " + data["layerNumber"] + ",bodyRegionId= " + data["bodyRegion"]); var logCol = db.GetCollection("DAImageLogs"); BodyRegionImage bodyRegionImageTolog = new BodyRegionImage(); bodyRegionImageTolog.bodyViewIndex = Convert.ToInt32(data["bodyViewIndex"]); bodyRegionImageTolog.layerNumber = Convert.ToInt32(data["layerNumber"]); bodyRegionImageTolog.bodyRegionId = Convert.ToInt32(data["bodyRegion"]); bodyRegionImageTolog.zoom = Convert.ToInt32(data["zoom"]); logCol.InsertOne(bodyRegionImageTolog); bodyRegionImageTolog = null; data = null; termList = null; systemTermMappingObject = null; systemTerms = null; } catch(Exception ex) { Console.WriteLine("AT " + DateTime.Now.ToString("yyyy-MM-ddThh:mm:sszzz") + "..Exception occured for imageName: " + data["imageName"].ToString() + ",bodyViewIndex= " + data["bodyViewIndex"] + ",layerNumber: " + data["layerNumber"] + ",bodyRegionId= " + data["bodyRegion"]+"... EXCEPTION= "+ex.Message+".. STACKTRACE= "+ex.StackTrace); } } public void AddToTermList(int termNumber, int systemNumber, int pixelLocation,Dictionary termList) { if (!termList.ContainsKey(termNumber)) { termList.Add(termNumber, new Term(termNumber, systemNumber, pixelLocation)); } else { bool isSystemNumberExisting = false; foreach (int sn in termList[termNumber].SystemNumbers) { if (systemNumber == sn) { isSystemNumberExisting = true; break; } } if (!isSystemNumberExisting) termList[termNumber].SystemNumbers.Add(systemNumber); bool isPixelLocationExisting = false; foreach (int pl in termList[termNumber].PixelLocations) { if (pixelLocation == pl) { isPixelLocationExisting = true; break; } } if (!isPixelLocationExisting) termList[termNumber].PixelLocations.Add(pixelLocation); } } static void Main(string[] args) { //string[] files = Directory.GetFiles("D:\\100-PROJECTS\\008-AIAHighlight.FileSave\\AIAHighlight\\AIAHighlight\\data", "*.json"); string[] files = Directory.GetFiles(@"C:\ADAM-AWS-FTP\ADAM-AWS\AIAHTML5\data\25", "*.json"); //string[] files = Directory.GetFiles(@"C:\ADAM-AWS-FTP\ADAM-AWS\AIAHTML5\data\BodyView4", "*.json"); Program program = new Program(); program.processFiles(files, 0, 19); program = null; } internal void processFiles(string[] files, int startIndex, int endIndex) { prepareArrayOfFiles(files, startIndex, endIndex); } internal void prepareArrayOfFiles(string[] files, int startIndex, int endIndex) { string[] filesToProcess = new string[20]; int processedFiles = 0; int j = 0; for (int i = startIndex; i <= endIndex; i++) { filesToProcess[j] = files[i]; j++; } Parallel.ForEach(filesToProcess, (currentFile) => { if (currentFile != null) { Console.WriteLine("currentFile = " + currentFile); using (StreamReader reader = new StreamReader(currentFile)) { getDataFromFileAndSaveInDB(currentFile); processedFiles++; } if (processedFiles == 20) { startIndex = endIndex + 1; endIndex = startIndex + 19; filesToProcess = null; //in case after adding 20, the endIndex reaches more than length of files array(total files) if (endIndex > files.Length - 1) { // int remainingFiles = endIndex - files.Length; endIndex = files.Length - 1; } if (endIndex < files.Length) { processFiles(files, startIndex, endIndex); } //} } } } ); } internal void getDataFromFileAndSaveInDB(string filePath) { if (File.Exists(filePath)) { string fileData; using (StreamReader reader = new StreamReader(filePath)) { fileData = reader.ReadToEnd(); JObject data = JObject.Parse(fileData); this.saveImageData(data); } } else { Console.WriteLine("FILE.NOT.FOUND AT" + ": " + filePath); } } //internal void getAlreadyInsertedLayer(string bodyviewId) //{ // Console.WriteLine("AT " + DateTime.Now.ToString("yyyy-MM-ddThh:mm:sszzz") + "inside getAlreadyInsertedLayer"); // int totalLayers = 0; // int [] totalBodyReigons =null; // List bodyRegions = new List(); // switch (Convert.ToInt32(bodyviewId)) // { // case 1: // totalLayers = 330; // totalBodyReigons = new int[6] {1,2,3,4,5,6}; // bodyRegions.Add(1); // bodyRegions.Add(2); // bodyRegions.Add(3); // bodyRegions.Add(4); // bodyRegions.Add(5); // bodyRegions.Add(6); // break; // case 2: // totalLayers = 288; // // totalBodyReigons = 6; // break; // case 3: // totalLayers = 127; // // totalBodyReigons = 6; // break; // case 4: // totalLayers = 186; // // totalBodyReigons = 6; // break; // case 5: // totalLayers = 229; // // totalBodyReigons = 6; // break; // case 6: // totalLayers = 289; // // totalBodyReigons = 6; // break; // case 7: // totalLayers = 130; // // totalBodyReigons = 6; // break; // case 8: // totalLayers = 186; // // totalBodyReigons = 6; // break; // case 9: // totalLayers = 64; // // totalBodyReigons = 6; // break; // case 10: // totalLayers = 72; // // totalBodyReigons = 6; // break; // case 11: // totalLayers = 64; // // totalBodyReigons = 6; // break; // case 12: // totalLayers = 72; // // totalBodyReigons = 6; // break; // } // var client = new MongoClient(); // var db = client.GetDatabase("AIADB_bodyviewId" + bodyviewId.ToString()); // //var db = client.GetDatabase("AIAImageDB"); // var col = db.GetCollection("DAImagesLogs"); // //var col = db.GetCollection("DAImages"); // FilterDefinition[] filterCondition = { // Builders.Filter.Eq("bodyViewIndex", Convert.ToInt32(bodyviewId))}; // dynamic list = col.Find(Builders.Filter.And(filterCondition)).Project(Builders.Projection.Include("layerNumber").Include("bodyRegionId")).ToList(); // //collection.Find(new BsonDocument()).Project(Builders.Projection.Include("Price").Exclude("_id")).ToListAsync(); // // return list; // Dictionary> layersToProcess = new Dictionary>(); // for (int i = 1; i <= totalLayers; i++) // { // List bodyRegionForThisLayer = new List(); // bodyRegionForThisLayer.Add(1); // bodyRegionForThisLayer.Add(2); // bodyRegionForThisLayer.Add(3); // bodyRegionForThisLayer.Add(4); // bodyRegionForThisLayer.Add(5); // bodyRegionForThisLayer.Add(6); // int previousLayerNumber = 0; // int existingLayerNumber = 0; // bool isLayerMatched = false; // foreach (dynamic lNumber in list) // { // existingLayerNumber = Convert.ToInt32((lNumber.GetValue("layerNumber")).ToString()); // if (existingLayerNumber == i) // { // isLayerMatched = true; // int existingBodyRegionId = Convert.ToInt32((lNumber.GetValue("bodyRegionId")).ToString()); // if (bodyRegionForThisLayer.Contains(existingBodyRegionId)) { // //remove from bodyRegions list as this is already present // bodyRegionForThisLayer.Remove(existingBodyRegionId); // } // previousLayerNumber = existingLayerNumber; // } // else // { // } // } // if(isLayerMatched) // layersToProcess.Add(previousLayerNumber, bodyRegionForThisLayer); // else // layersToProcess.Add(i, bodyRegionForThisLayer); // } // Console.WriteLine("AT " + DateTime.Now.ToString("yyyy-MM-ddThh:mm:sszzz") + "after preparing list to process"); // makeFileNameAndProcess(layersToProcess,Convert.ToInt32(bodyviewId)); //} //internal void makeFileNameAndProcess(Dictionary> filesToProcess, int BodyViewIndex) //{ // Console.WriteLine("AT " + DateTime.Now.ToString("yyyy-MM-ddThh:mm:sszzz") + "in makeFileNameAndProcess"); // foreach (KeyValuePair> pair in filesToProcess) // { // foreach (int item in pair.Value) // { // string fileName = "BodyView" + BodyViewIndex + "_layer" + pair.Key + "_bodyRegionId" + item+".json"; // //string filePath = @"D:\100-PROJECTS\008-AIAHighlight.FileSave\AIAHighlight\AIAHighlight\data\" + fileName; // string filePath = @"C:\ADAM-AWS-FTP\ADAM-AWS\AIAHTML5\data\" + fileName; // getDataFromFileAndSaveInDB(filePath, BodyViewIndex); // } // } //} //internal void processFiles(string[] files,int bodyviewId) //{ // //without multithreading // foreach(string filename in files) // { // Console.WriteLine("AT " + DateTime.Now.ToString("yyyy-MM-ddThh:mm:sszzz") + "in processFiles for filename"); // getDataFromFileAndSaveInDB(filename, bodyviewId); // } //} //internal void getDataFromFileAndSaveInDB(string filePath,int bodyViewId) //{ // if (File.Exists(filePath)) // { // string fileData; // using (StreamReader reader = new StreamReader(filePath)) // { // fileData = reader.ReadToEnd(); // JObject data = JObject.Parse(fileData); // this.saveImageData(data, bodyViewId); // } // } // else // { // Console.WriteLine("FILE.NOT.FOUND AT" + // ": " + filePath); // } //} } }