diff --git a/README.md b/README.md index b44751fb..0541b5fb 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,34 @@ -# Data Pre-processor -De-constructing regular pdf's,docx format based information into structured JSON format. - ---- - -## How to Contribute - -To contribute to our documentation: - -1. **Fork the Repository:** Click the "Fork" button at the top right of this repository to create a copy in your GitHub account. 🍴 - -2. **Clone Your Fork:** Clone the forked repository to your local machine using Git. 🖥️ - - ```bash - git clone https://github.com//data_preprocessor.git - ``` - -3. **Create a Branch:** Create a new branch for your contribution. 🌿 - - ```bash - git checkout -b - ``` -4. **Virtual Evnvironment:** Create necessary virtual environtment or docker container; prefer if you look into docker and stuff. -5. Use Git CLI to add your files and track it. -6. Once pushed to your branch give a pull request. - ---- +# Project Report: Data Pre-processing + +## 1. Initial Approach: Using Regex and Spacy for Data Extraction +The first approach involved using a combination of **regular expressions (regex)** and **Spacy** for data extraction. The plan was to: +- Use **regex patterns** to identify and extract the headings from the data. +- Use **Spacy**, a powerful natural language processing library, to extract the subheadings based on named entity recognition and other language features. + +This method was partially effective but had limitations in identifying all necessary data due to inconsistencies in the formatting of headings and subheadings. + +## 2. Extracting Data Based on Font Styles +The next idea involved attempting to extract information based on the **font styles** in the document. For example: +- **Blue-colored text** was identified as main headings. +- **Black-colored text** was identified as subheadings. + +This approach relied on font color and style metadata, but it proved challenging to implement consistently across different documents, as not all documents had well-defined font styling for hierarchical structures. + +## 3. Final Approach: Regex and Section-based Functions +The final approach used **regex** for pattern matching and involved creating **separate functions** for each section. For instance: +- A function was designed to extract information specifically from the **Identification** section of a document, such as: + +```python +def format_identification(section_text): + lines = section_text.splitlines() + identification_data = {} + for line in lines: + if "Product Name" in line and ":" in line: + identification_data["ProductName"] = line.split(":")[1].strip() + elif "Product Name" in line and ":" not in line: + identification_data["ProductName"] = line.replace("Product Name", "").strip() + elif "Cat No." in line and ":" in line: + identification_data["CatNumbers"] = [item.strip() for item in line.split(":")[1].strip().split(";")] + elif "Cat No." in line and ":" not in line: + identification_data["CatNumbers"] = [item.strip() for item in line.replace("Cat No.", "").strip().split(";")] + return identification_data diff --git a/data-processing/data.pdf b/data-processing/data.pdf new file mode 100644 index 00000000..3663e61e Binary files /dev/null and b/data-processing/data.pdf differ diff --git a/data-processing/main.py b/data-processing/main.py new file mode 100644 index 00000000..7c3adba1 --- /dev/null +++ b/data-processing/main.py @@ -0,0 +1,136 @@ +import re +import json + + +extracted_text = """ + +""" + + +patterns = { + "Identification": r"--- 1\. Identification ---\n(.*?)--- 2\. Hazard", + "HazardIdentification": r"--- 2\. Hazard\(s\) identification ---\n(.*?)--- 3\. Composition", + "Composition": r"--- 3\. Composition/Information on Ingredients ---\n(.*?)--- 4\. First-aid", + "FirstAidMeasures": r"--- 4\. First-aid measures ---\n(.*?)--- 5\. Fire-fighting", + "FireFightingMeasures": r"--- 5\. Fire-fighting measures ---\n(.*?)--- 6\. Accidental release", + "AccidentalRelease": r"--- 6\. Accidental release measures ---\n(.*?)--- 7\. Handling", + "HandlingStorage": r"--- 7\. Handling and storage ---\n(.*?)--- 8\. Exposure controls", + "ExposureControls": r"--- 8\. Exposure controls / personal protection ---\n(.*?)--- 9\. Physical", + "PhysicalChemicalProperties": r"--- 9\. Physical and chemical properties ---\n(.*?)--- 10\. Stability", + "StabilityReactivity": r"--- 10\. Stability and reactivity ---\n(.*?)--- 11\. Toxicological", + "ToxicologicalInformation": r"--- 11\. Toxicological information ---\n(.*?)--- 12\. Ecological", + "EcologicalInformation": r"--- 12\. Ecological information ---\n(.*?)--- 13\. Disposal", + "DisposalConsiderations": r"--- 13\. Disposal considerations ---\n(.*?)--- 14\. Transport", + "TransportInformation": r"--- 14\. Transport information ---\n(.*?)--- 15\. Regulatory", + "RegulatoryInformation": r"--- 15\. Regulatory information ---\n(.*?)--- 16\. Other", + "OtherInformation": r"--- 16\. Other information ---\n(.*)" +} + + + +def extract_section(text, pattern): + match = re.search(pattern, text, re.DOTALL) + return match.group(1).strip() if match else "Section not found" + + + +def format_identification(section_text): + lines = section_text.splitlines() + identification_data = {} + + for line in lines: + if "Product Name" in line and ":" in line: + identification_data["ProductName"] = line.split(":")[1].strip() + elif "Product Name" in line and ":" not in line: + identification_data["ProductName"] = line.replace("Product Name", "").strip() + elif "Cat No." in line and ":" in line: + identification_data["CatNumbers"] = [item.strip() for item in line.split(":")[1].strip().split(";")] + elif "Cat No." in line and ":" not in line: + identification_data["CatNumbers"] = [item.strip() for item in line.replace("Cat No.", "").strip().split(";")] + + return identification_data + + +def format_hazard_identification(section_text): + hazard_data = {} + lines = section_text.splitlines() + + for line in lines: + if "Flammable liquids" in line: + hazard_data["FlammableLiquids"] = line.split("Category")[1].strip() + elif "Eye Damage" in line: + hazard_data["EyeDamage"] = line.split("Category")[1].strip() + elif "Specific target organ toxicity" in line: + hazard_data["SpecificTargetOrganToxicity"] = line.split("Category")[1].strip() + + return hazard_data + +def format_composition(section_text): + composition_data = [] + lines = section_text.splitlines() + + for line in lines: + if "Component" in line or "CAS No" in line or "Weight %" in line: + continue + parts = line.split() + if len(parts) >= 3: + composition_data.append({ + "ChemicalName": parts[0], + "CASNumber": parts[1], + "Concentration": parts[2] + }) + + return composition_data + + +def format_first_aid_measures(section_text): + first_aid_data = {} + lines = section_text.splitlines() + + for line in lines: + if "Eye Contact" in line: + first_aid_data["EyeContact"] = line.split("Eye Contact")[1].strip() + elif "Skin Contact" in line: + first_aid_data["SkinContact"] = line.split("Skin Contact")[1].strip() + elif "Inhalation" in line: + first_aid_data["Inhalation"] = line.split("Inhalation")[1].strip() + elif "Ingestion" in line: + first_aid_data["Ingestion"] = line.split("Ingestion")[1].strip() + + return first_aid_data + + +def format_fire_fighting_measures(section_text): + fire_fighting_data = {} + lines = section_text.splitlines() + + for line in lines: + if "Suitable Extinguishing Media" in line: + fire_fighting_data["ExtinguishingMedia"] = line.split("Media")[1].strip() + elif "Flash Point" in line: + fire_fighting_data["FlashPoint"] = line.split("Flash Point")[1].strip() + elif "Autoignition Temperature" in line: + fire_fighting_data["AutoignitionTemperature"] = line.split("Autoignition Temperature")[1].strip() + + return fire_fighting_data + + +json_data = {} + + +identification_text = extract_section(extracted_text, patterns["Identification"]) +json_data["Identification"] = format_identification(identification_text) + +hazard_identification_text = extract_section(extracted_text, patterns["HazardIdentification"]) +json_data["HazardIdentification"] = format_hazard_identification(hazard_identification_text) + +composition_text = extract_section(extracted_text, patterns["Composition"]) +json_data["Composition"] = format_composition(composition_text) + +first_aid_measures_text = extract_section(extracted_text, patterns["FirstAidMeasures"]) +json_data["FirstAidMeasures"] = format_first_aid_measures(first_aid_measures_text) + +fire_fighting_measures_text = extract_section(extracted_text, patterns["FireFightingMeasures"]) +json_data["FireFightingMeasures"] = format_fire_fighting_measures(fire_fighting_measures_text) + +print(json_data) \ No newline at end of file