Индексации хранилища текстов на основе естественно-языковой адресации

void UpdateDocument(Document document, bool IsTextChanged);

[OperationContract]

[WebGet(ResponseFormat = WebMessageFormat.Json, UriTemplate = "DeleteDocument?id={documentId}")]

void DeleteDocument(int documentId);

[OperationContract]

[WebInvoke(Method = "POST",

BodyStyle = WebMessageBodyStyle.Wrapped,

ResponseFormat = WebMessageFormat.Json,

UriTemplate = "UpdateCorpus")]

void UpdateCorpus(Corpus corpus);

[OperationContract]

[WebGet(ResponseFormat = WebMessageFormat.Json, UriTemplate = "DeleteCorpus?id={corpusId}")]

void DeleteCorpus(int corpusId);

[OperationContract]

[WebInvoke(Method = "POST",

BodyStyle = WebMessageBodyStyle.Wrapped,

ResponseFormat = WebMessageFormat.Json,

UriTemplate = "FindDocuments")]

Document[] FindDocuments(string Name, int corpusId);

[OperationContract]

[WebInvoke(Method = "POST",

BodyStyle = WebMessageBodyStyle.Wrapped,

ResponseFormat = WebMessageFormat.Json,

UriTemplate = "FindText")]

int[] FindText(string Text, int corpusId);

}

}

namespace PaperCatPortal

{

public class CorporaStorageService : ICorporaStorageService

{

static HashSet<string> stopwords = new HashSet<string>() { "a", "about", "above", "after", "again", "against", "all", "am", "an", "and", "any", "are", "aren't", "as", "at", "be", "because", "been", "before", "being", "below", "between", "both", "but", "by", "can't", "cannot", "could", "couldn't", "did", "didn't", "do", "does", "doesn't", "doing", "don't", "down", "during", "each", "few", "for", "from", "further", "had", "hadn't", "has", "hasn't", "have", "haven't", "having", "he", "he'd", "he'll", "he's", "her", "here", "here's", "hers", "herself", "him", "himself", "his", "how", "how's", "i", "i'd", "i'll", "i'm", "i've", "if", "in", "into", "is", "isn't", "it", "it's", "its", "itself", "let's", "me", "more", "most", "mustn't", "my", "myself", "no", "nor", "not", "of", "off", "on", "once", "only", "or", "other", "ought", "our", "ours”, “ourselves", "out", "over", "own", "same", "shan't", "she", "she'd", "she'll", "she's", "should", "shouldn't", "so", "some", "such", "than", "that", "that's", "the", "their", "theirs", "them", "themselves", "then", "there", "there's", "these", "they", "they'd", "they'll", "they're", "they've", "this", "those", "through", "to", "too", "under", "until", "up", "very", "was", "wasn't", "we", "we'd", "we'll", "we're", "we've", "were", "weren't", "what", "what's", "when", "when's", "where", "where's", "which", "while", "who", "who's", "whom", "why", "why's", "with", "won't", "would", "wouldn't", "you", "you'd", "you'll", "you're", "you've", "your", "yours", "yourself", "yourselves" };

public int AddDocument(Document document)

{

if (document == null)

throw new Exception("Параметр не может быть null");

using (var context = new PaperCatModelContainer())

{

DocumentModel documentModel = new DocumentModel(document, context);

if (documentModel.Сorpus == null)

throw new Exception("Корпуса c таким id не существует");

context.DocumentModelSet.Add(documentModel);

context.SaveChanges();

document.DocumentId = documentModel.DocumentId;

}

DocumentClient client = GetDocumentClient();

client.CreateDocumentAsync(UriFactory.CreateDocumentCollectionUri(Properties.Resources.DataBaseId, Properties.Resources.DocumentsCollectionId), new AzureDocument(document)).Wait();

Task.Run(() => AddDocumentToIndexStructure(document));

return document.DocumentId;

}

private void AddDocumentToIndexStructure(Document document)

{

string text = document.FlatText;

text = GetTokens(text);

NLAStructureHelper nlaHelper = new NLAStructureHelper(GetDocumentClient(), document.CorpusId);

foreach (string item in text.Split(' ').Distinct())

{

nlaHelper.Add(item, document.DocumentId);

}

}

/// <summary>

/// Очищает от знаков и стоп-слов, стеммит

/// </summary>

/// <param name="text"></param>

/// <returns></returns>

private string GetTokens(string text)

{

string newText = "";

Regex rgx = new Regex("[^a-zA-Zа-яА-Я ]");

text = rgx.Replace(text, "");

string[] words = text.Split(' ');

Stemmer stemmer = new Stemmer();

for (int i = 0; i < words.Length; i++)

{

//убираем стоп-слова

if (words[i].Length != 1 && !stopwords.Contains(words[i]))

{

words[i] = stemmer.stem(words[i]);

newText += " " + words[i];

}

}

return newText.Trim(' ');

}

public int AddUser(User user)

{

if (user == null)

throw new Exception("Параметр не может быть null");

using (var context = new PaperCatModelContainer())

{

UserModel userModel = new UserModel(user);

if (IsThisLoginExist(user.Login, context))

{

throw new Exception("Пользователь с таким логином уже существует");

}

context.UserModelSet.Add(userModel);

context.SaveChanges();

return userModel.UserId;

}

}

private static bool IsThisLoginExist(string sLogin, PaperCatModelContainer context, int userid = 0)

{

var us = context.UserModelSet.Where(u => u.Login == sLogin && u.UserId != userid).FirstOrDefault();

return us != null;

}

public User AutorizeUser(string Login, string Password)

{

using (var context = new PaperCatModelContainer())

{

var userModel = context.UserModelSet.Where(u => u.Login == Login && u.Password == Password).FirstOrDefault();

if (userModel == null) return null; //такого пользователя нет

return userModel.GetUserFromModel();

}

}

public void DeleteCorpus(int corpusId)

{

using (var context = new PaperCatModelContainer())

{

DeleteCorpusInContext(corpusId, context);

context.SaveChanges();

}

}

private void DeleteCorpusInContext(int corpusId, PaperCatModelContainer context)

{

var DocumentList = context.DocumentModelSet.Where(d => d.Сorpus.CorpusId == corpusId).ToList();

DocumentClient client = GetDocumentClient();

foreach (var item in DocumentList)

{

DeleteDocumentInContext(item.DocumentId, context);

Task.Run(() => DeleteDocumentFromIndexStructure(item.DocumentId, corpusId));

string SelfLink = GetDocumentSelfLink(item.DocumentId, client);

if (SelfLink != null)

client.DeleteDocumentAsync(SelfLink).Wait();

}

CorpusModel corpusModel = context.CorpusModelSet.Find(corpusId);

if (corpusModel != null)

{

context.CorpusModelSet.Remove(corpusModel);

}

}

/// <summary>

//// Возвращает id корпуса, из которого был удален документ

/// </summary>

/// <param name="documentId"></param>

/// <param name="context"></param>

/// <returns></returns>

private int DeleteDocumentInContext(int documentId, PaperCatModelContainer context)

{

int corpusId = -1;

DocumentModel documentModel = context.DocumentModelSet.Find(documentId);

if (documentModel != null)

{

corpusId = documentModel.Сorpus.CorpusId;

context.DocumentModelSet.Remove(documentModel);

}

return corpusId;

}

public void DeleteDocument(int documentId)

{

int corpusid;

using (var context = new PaperCatModelContainer())

{

corpusid = DeleteDocumentInContext(documentId, context);

context.SaveChanges();

}

Task.Run(() => DeleteDocumentFromIndexStructure(documentId, corpusid));

DocumentClient client = GetDocumentClient();

string SelfLink = GetDocumentSelfLink(documentId, client);

if (SelfLink != null)

client.DeleteDocumentAsync(SelfLink).Wait();

}

private static string GetDocumentSelfLink(int documentId, DocumentClient client)

{

return client.CreateDocumentQuery(UriFactory.CreateDocumentCollectionUri(Properties.Resources.DataBaseId, Properties.Resources.DocumentsCollectionId)).AsEnumerable().Where(d => d.GetPropertyValue<int>("DocumentId") == documentId).FirstOrDefault()?.SelfLink;

}

private static DocumentClient GetDocumentClient()

{

return new DocumentClient(new Uri(Properties.Resources.EndpointUrl), Properties.Resources.PrimaryKey);

}

private void DeleteDocumentFromIndexStructure(int documentId, int corpusId)

{

NLAStructureHelper nlaHelper = new NLAStructureHelper(GetDocumentClient(), corpusId, false);

nlaHelper.Delete(documentId);

}

public void DeleteUser(int userId)

{

using (var context = new PaperCatModelContainer())

{

var corpusList = context.CorpusModelSet.Where(c => c.User.UserId == userId).ToList();

foreach (var item in corpusList)

{

DeleteCorpusInContext(item.CorpusId, context);

}

UserModel UserModel = context.UserModelSet.Find(userId);

if (UserModel == null)

return;

context.UserModelSet.Remove(UserModel);

context.SaveChanges();

}

}

public Document[] FindDocuments(string Name, int corpusId)

{

List<DocumentModel> DocumentModelList;

using (var context = new PaperCatModelContainer())

{

DocumentModelList = context.DocumentModelSet.Where(d => d.Сorpus.CorpusId == corpusId && d.Name.ToLower().Contains(Name.ToLower())).ToList();

Document[] documents = new Document[DocumentModelList.Count()];

for (int i = 0; i < DocumentModelList.Count(); i++)

{

documents[i] = GetFullDocument(DocumentModelList[i]);

}

return documents;

}

}

private static Document GetDocumentNonRelationalProperties(Document document, DocumentClient client)

{

Microsoft.Azure.Documents.Document AzureDocument = client.CreateDocumentQuery(UriFactory.CreateDocumentCollectionUri(Properties.Resources.DataBaseId, Properties.Resources.DocumentsCollectionId)).AsEnumerable().Where(d => d.GetPropertyValue<int>("DocumentId") == document.DocumentId).FirstOrDefault();

document.Annotation = AzureDocument.GetPropertyValue<string>("Annotation");

document.FlatText = AzureDocument.GetPropertyValue<string>("FlatText");

document.HTMLstring = AzureDocument.GetPropertyValue<string>("HTMLstring");

return document;

}

public int[] FindTextSimple(string Text, int corpusId)

{

int[] document = null;

var documents = GetDocuments(corpusId);

Text = GetTokens(Text);

foreach (var item in Text.Split(' '))

{

List<int> d = new List<int>();

foreach (var ditem in documents)

{

string str = GetTokens(ditem.FlatText);

if (str.Contains(" " + item + " "))

d.Add(ditem.DocumentId);

}

if (document == null)

{

document = d.ToArray();

}

else

document.Intersect(d.ToArray());

}

return document;

}

public int[] FindText(string Text, int corpusId)

{

//Последовательность вхождения не смотриться, просто наличие ключевых слов в тексте

Text = GetTokens(Text);

int[] documentIds = null;

NLAStructureHelper nlaHelper = new NLAStructureHelper(GetDocumentClient(), corpusId, false);

if (nlaHelper == null)

return null;

foreach (var item in Text.Split(' '))

{

if (documentIds == null)

documentIds = nlaHelper.FindWordDocumentIds(item).ToArray();

else

documentIds.Intersect(nlaHelper.FindWordDocumentIds(item));

}

return documentIds;

//Document[] documents = new Document[documentIds.Count()];

//for (int i = 0; i < documentIds.Count(); i++)

//{

// documents[i] = GetDocument(documentIds[i]);

//}

//return documents;

}

public Corpus[] GetCorpora(int UserId)

{

using (var context = new PaperCatModelContainer())

{

List<CorpusModel> corpusModels;

if (UserId != 0)

corpusModels = context.CorpusModelSet.Where(c => c.User.UserId == UserId).ToList();

else

corpusModels = context.CorpusModelSet.Where(c => c.User == null).ToList();

Corpus[] corpuses = new Corpus[corpusModels.Count()];

for (int i = 0; i < corpusModels.Count(); i++)

{

corpuses[i] = corpusModels[i].GetCorpusFromModel();

}

return corpuses;

}

}

public Document GetDocument(int DocumentId)

{

DocumentModel doc;

using (var context = new PaperCatModelContainer())

{

doc = context.DocumentModelSet.Find(DocumentId);

if (doc == null)

return null;

return GetFullDocument(doc);

}

}

private static Document GetFullDocument(DocumentModel document)

{

return GetDocumentNonRelationalProperties(document.GetDocumentFromModel(), GetDocumentClient());

}

public Document[] GetDocuments(int CorpusId)

{

using (var context = new PaperCatModelContainer())

{

var documentList = context.DocumentModelSet.Where(d => d.Сorpus.CorpusId == CorpusId).ToList();

Document[] documents = new Document[documentList.Count];

for (int i = 0; i < documentList.Count; i++)

{

documents[i] = GetFullDocument(documentList[i]);

}

return documents;

}

}

public int MakeCommonCorpus(string Name, Document[] documents)

{

return MakeCorpus(Name, documents, 0);

}

public int MakeCorpus(string Name, Document[] documents, int UserId)

{

using (var context = new PaperCatModelContainer())

{

CorpusModel corpusModel = new CorpusModel()

{

CorpusName = Name,

User = context.UserModelSet.FirstOrDefault(u => u.UserId == UserId)

};

context.CorpusModelSet.Add(corpusModel);

context.SaveChanges();

if (documents != null)

foreach (var item in documents)

{

item.CorpusId = corpusModel.CorpusId;

AddDocument(item);

}

return corpusModel.CorpusId;

}

}

public void UpdateCorpus(Corpus corpus)

{

if (corpus == null)

throw new Exception("Параметр не может быть null");

using (var context = new PaperCatModelContainer())

{

var corpusModel = context.CorpusModelSet.Find(corpus.CorpusId);

if (corpusModel == null)

{

throw new Exception("Такого корпуса не существует");

}

corpusModel.CorpusName = corpus.CorpusName;

context.SaveChanges();

}

}

public void UpdateDocument(Document document, bool IsTextChanged)

{

if (document == null)

throw new Exception("Параметр не может быть null");

using (var context = new PaperCatModelContainer())

{

var documentModel = context.DocumentModelSet.Find(document.DocumentId);

if (documentModel == null)

{

throw new Exception("Такого документа не существует");

}

documentModel.Name = document.Name;

documentModel.OriginalFile = document.OriginalFile;

document.CorpusId = documentModel.Сorpus.CorpusId; //документ остается в том же корпусе

context.SaveChanges();

}

DocumentClient client = GetDocumentClient();

string selfLink = GetDocumentSelfLink(document.DocumentId, client);

if (IsTextChanged)

{

Task.Run(() => { DeleteDocumentFromIndexStructure(document.DocumentId, document.CorpusId); AddDocumentToIndexStructure(document); } );

}

if (selfLink != null)

client.ReplaceDocumentAsync(selfLink, new AzureDocument(document)).Wait();

}

public void UpdateUser(User user)

{

if (user == null)

throw new Exception("Параметр не может быть null");

using (var context = new PaperCatModelContainer())

{

var userModel = context.UserModelSet.Find(user.UserId);

if (userModel == null)

{

throw new Exception("Такого пользователя не существует");

}

userModel.FIO = user.FIO;

if (IsThisLoginExist(user.Login, context, user.UserId))

{

throw new Exception("Другой пользователь с таким логином уже существует");

}

userModel.Login = user.Login;

userModel.Password = user.Password;

userModel.Email = user.Email;

context.SaveChanges();

}

}

}

}

{

public class NLAStructureHelper

{

private const int nSymbols = 3;

string HeadSelfLink = "";

NLAStructure Head;

DocumentClient documentClient;

/// <summary>

/// Поиск осуществляется отдельно для каждого корпуса

/// </summary>

/// <param name="client"></param>

/// <param name="Corpusid"></param>

public NLAStructureHelper(DocumentClient client, int Corpusid, bool isForceCreate = true)

{

documentClient = client;

Head = LoadHead(Corpusid, isForceCreate);

}

/// <summary>

/// Добавить id документа к данному слову

/// </summary>

/// <param name="word"></param>

/// <param name="documentId"></param>

public void Add(string word, int documentId)

{

NLAStructure structure = Head;

string CurrentStructureLink = HeadSelfLink;

for (int i = 0; i < word.Length; i += nSymbols)

{

string key = word.Length < i + nSymbols ? word.Substring(i) : word.Substring(i, nSymbols);

IndexStructureObject obj;

if (structure.Dictionary.ContainsKey(key))

obj = structure.Dictionary[key];

else

{

obj = new IndexStructureObject();

structure.Dictionary.Add(key, obj);

}

if (i + nSymbols >= word.Length) //дошли до нужного листа

{

obj.DocumentIds.Add(documentId);

SaveChanges(CurrentStructureLink, structure);

}

else

{

NLAStructure oldNLAStructure = structure;

structure = GetNextStructure(obj);

SaveChanges(CurrentStructureLink, oldNLAStructure);

CurrentStructureLink = obj.NextStructureSelfLink;

}

}

}

/// <summary>

//// Удалить все вхождения id документа из структуры

/// </summary>

/// <param name="documentId"></param>

public void Delete(int documentId)

{

Delete(documentId, Head, HeadSelfLink);

}

private bool Delete(int documentId, NLAStructure NLAStructure, string CurrentStructureLink)

{

if (NLAStructure != null)

{

var TemtDictionary = new Dictionary<string, IndexStructureObject>(NLAStructure.Dictionary);

foreach (var item in TemtDictionary)

{

IndexStructureObject iso = item.Value;

NLAStructure nextNLAStructure = GetNextStructure(iso, false);

if (nextNLAStructure != null)

if (Delete(documentId, nextNLAStructure, iso.NextStructureSelfLink))

iso.NextStructureSelfLink = "";

iso.DocumentIds.Remove(documentId);

if (iso.NextStructureSelfLink == "" && iso.DocumentIds.Count == 0)

{

NLAStructure.Dictionary.Remove(item.Key);

}

}

}

if (NLAStructure.Dictionary.Count == 0)

{

documentClient.DeleteDocumentAsync(CurrentStructureLink);

return true;

}

SaveChanges(CurrentStructureLink, NLAStructure);

return false;

}

private NLAStructure GetNextStructure(IndexStructureObject indexStructureObject, bool isForcedCreate = true)

{

if (indexStructureObject.NextStructureSelfLink != "")

{

Document doc = documentClient.ReadDocumentAsync(indexStructureObject.NextStructureSelfLink).Result;

NLAStructure Structure = (dynamic)doc;

return Structure;

}

else if (isForcedCreate)

{

NLAStructure newStructure = new NLAStructure();

Document doc = documentClient.CreateDocumentAsync(UriFactory.CreateDocumentCollectionUri(Properties.Resources.DataBaseId, Properties.Resources.IndicesCollectionId), newStructure).Result;

indexStructureObject.NextStructureSelfLink = doc.SelfLink;

return newStructure;

}

else return null;

}

private void SaveChanges(string StructureLink, NLAStructure structure)

{

documentClient.ReplaceDocumentAsync(StructureLink, structure).Wait();

}

private NLAStructure LoadHead(int CorpusId, bool isForceCreate = true)

{

Uri documentCollectionUri = UriFactory.CreateDocumentCollectionUri(Properties.Resources.DataBaseId, Properties.Resources.IndicesCollectionId);

Microsoft.Azure.Documents.Document doc = CorpusId != 0 ?

documentClient.CreateDocumentQuery(documentCollectionUri).AsEnumerable().Where(d => d.GetPropertyValue<int>("CorpusId") == CorpusId).FirstOrDefault()

: null ;

NLAHeadDocument Structure = null;

if (doc != null)

{

Structure = (dynamic)doc;

HeadSelfLink = doc.SelfLink;

}

else if (isForceCreate)

{

Structure = new NLAHeadDocument(CorpusId);

doc = documentClient.CreateDocumentAsync(documentCollectionUri, Structure).Result;

HeadSelfLink = doc.SelfLink;

}

return Structure;

}

internal List<int> FindWordDocumentIds(string word)

{

NLAStructure structure = Head;

IndexStructureObject obj = null;

for (int i = 0; i < word.Length; i += nSymbols)

{

string key = word.Length < i + nSymbols ? word.Substring(i) : word.Substring(i, nSymbols);

if (structure != null && structure.Dictionary.ContainsKey(key))

{

obj = structure.Dictionary[key];

structure = GetNextStructure(obj, false);

}

else return new List<int>();

}

if (obj != null)

return obj.DocumentIds;

return new List<int>();

}

}

}

Размещено на Allbest.ru