diff --git a/api/data/filters.sql b/api/data/filters.sql index a23b9d41..dd0fca2a 100644 --- a/api/data/filters.sql +++ b/api/data/filters.sql @@ -1,44 +1,53 @@ DELETE FROM public.page_filters; + -- Sections in the survey + -- Section 1 - General Information -INSERT INTO public.page_filters VALUES ('organisation-by-sector', 'Agriculture;Forestry;Both'); -INSERT INTO public.page_filters VALUES ('type-of-stakeholder', 'Farmer/agricultural producers;Forester;Forest owner;Forest operator;Forest product processor;Farming association;Farming cooperative;Forest industry association;Trade association;NGO/Advisory Group;Data association/organisation/coalition;Data provider;Platform provider;Service/Information provider;Digital technology provider; Research institutes and research networks; National and European networks'); -INSERT INTO public.page_filters VALUES ('eu-member-state', 'Austria;Belgium;Bulgaria;Croatia;Cyprus;Czechia;Denmark;Finland;France;Germany;Greece;Hungary;Italy;Latvia;Lithuania;Luxembourg;Malta;Poland;Portugal;Romania;Slovakia;Slovenia;Spain;Sweden;The Netherlands'); -INSERT INTO public.page_filters VALUES ('in-agriculture', 'Crop cultivation- grains;Crop cultivation- vegetables;Crop cultivation- legumes;Crop cultivation- fruits;Plant propagation;Livestock farming - meat;Livestock farming- dairy;Mixed Farming (crops and animal);Agricultural machinery and equipment services;Crop services (monitoring);Farm management services;Post-harvest handling services '); -INSERT INTO public.page_filters VALUES ('in-forestry', 'Reforestation;Forest conservation - thinning, pruning, weed & pest control;Felling;Transportation of logs;Non-Timber Forest Products (NTFPs);Forest Fire Management;Forestry inventory and mapping;Other'); --- Section 2 - Workforce information -INSERT INTO public.page_filters VALUES ('organisation-size', 'Small-scale/Local;Medium-scale/Local-National;Large Scale/National-International'); -INSERT INTO public.page_filters VALUES ('education-level', 'High education;Advanced education;Technical and Vocational Education;Basic education'); -INSERT INTO public.page_filters VALUES ('experience-level', 'Experts;Mid-level professionals;Early-career/Entry level'); -INSERT INTO public.page_filters VALUES ('training-on-digital-technologies', 'Training programmes;Workshops;Seminars;Other'); --- Section 3 - Governance model -INSERT INTO public.page_filters VALUES ('governance-model-type', 'Traditional/Subsistence Agriculture or Forestry;Cooperative Agriculture or Forestry;Corporate Agriculture or Forestry;Contract Farming or Forestry;Community Supported Agriculture or Forestry;Land trust and conservation Agriculture or Forestry;Other'); --- Section 4 - Adoption of Digital Technologies and Technology Integration -INSERT INTO public.page_filters VALUES ('technology-type-agriculture', 'Precision Farming;Farm Management Information Systems;Automated machinery and robotics;Smart Irrigation systems;Monitoring and tracking of livestock/crops;Smart-agri apps;Other (please specify)'); +INSERT INTO public.page_filters VALUES ('sector', 'Agriculture;Forestry;Both'); +INSERT INTO public.page_filters VALUES ('type-of-stakeholder', 'Farmer/agricultural producers;Forester;Forest owner;Forest operator;Forest product processor;Farming association;Farming cooperative;Forestry association;Forest industry association;Trade association;NGO/Advisory Group;Data association/organisation/coalition;Data provider;Infrastructure provider;Platform provider;Service/Information provider;Digital technology provider;Research institutes and research networks;National and European networks'); +INSERT INTO public.page_filters VALUES ('location-country-region', 'Austria;Belgium;Bulgaria;Croatia;Cyprus;Czechia;Denmark;Estonia;Finland;France;Germany;Greece;Hungary;Ireland;Italy;Latvia;Lithuania;Luxembourg;Malta;Poland;Portugal;Romania;Slovakia;Slovenia;Spain;Sweden;The Netherlands'); +INSERT INTO public.page_filters VALUES ('primary-area-of-operation-in-agriculture', 'Crop cultivation- grains;Crop cultivation- vegetables;Crop cultivation- legumes;Crop cultivation- fruits;Plant propagation;Livestock farming - meat;Livestock farming- dairy;Livestock farming- other;Mixed Farming (crops and animal);Agricultural machinery and equipment services;Crop services (monitoring);Farm management services;Post-harvest handling services;Other'); +INSERT INTO public.page_filters VALUES ('primary-area-of-operation-in-forestry', 'Reforestation;Forest conservation - thinning, pruning, weed & pest control;Felling;Transportation of logs;Non-Timber Forest Products (NTFPs);Forest Fire Management;Forestry inventory and mapping;Wildlife management;Other'); +INSERT INTO public.page_filters VALUES ('organic-farming-operation', 'Yes;No;Don''t know'); + +-- Section 2 - Governance Model +INSERT INTO public.page_filters VALUES ('governance-model-impact', 'Facilitated;Hindered;No impact'); +INSERT INTO public.page_filters VALUES ('regulatory-considerations', 'Yes;No;Don''t know'); + +-- Section 3 - Adoption of Digital Technologies and Technology Integration +INSERT INTO public.page_filters VALUES ('digital-technologies-integrated', 'Yes;No;Don''t know'); +INSERT INTO public.page_filters VALUES ('technology-type-agriculture', 'Precision Farming;Farm Management Information Systems;Automated machinery and robotics;Smart Irrigation systems;Monitoring and tracking of livestock/crops;Smart-agri apps;Other (please specify)'); INSERT INTO public.page_filters VALUES ('technology-type-forestry', 'Forest Inventory Management Software;Drones for Forest Monitoring;Autonomated machinery and robotics;Forest Fire Prediction and Monitoring Systems;Other (please specify)'); +INSERT INTO public.page_filters VALUES ('goals-or-challenges', 'Yes;No;Don''t know'); INSERT INTO public.page_filters VALUES ('level-of-digitalization', '1;2;3;4;5'); -INSERT INTO public.page_filters VALUES ('primary-functions', 'On-farm activities;Production phase;Monitoring;Supply chain optimisation;Decision-making;Planning and management;Crop health and desease detection;Harvesting and distribution;Data management'); +INSERT INTO public.page_filters VALUES ('primary-functions', 'On-farm activities;Production phase;Monitoring;Supply chain optimisation;Decision-making;Planning and management;Crop health and disease detection;Harvesting and distribution;Data management'); INSERT INTO public.page_filters VALUES ('adoption-level', 'Fully integrated;Advanced;Preliminary'); --- Section 5 - Technology performance -INSERT INTO public.page_filters VALUES ('advantages', 'Increased efficiency and productivity;Improved decision-making;Efficient Resource Allocation;Traceability and Transparency;Early Detection of Issues;Economic Benefits;Smart Irrigation and Water Conservation;Improved Forest Management;Enhanced Safety and Monitoring;Other'); -INSERT INTO public.page_filters VALUES ('limitations-or-challenges', 'High costs (initial and/or operational);Complexity;Skills and Training Requirements;Limited Accessibility and Connectivity;Interoperability Issues;Data Privacy and Security Concerns;Resistance to Change;Other;Farm-level data;Earth Observation (EO) data;Environmental data;Socio-economic data;Supply chain data;Research and Development data'); -INSERT INTO public.page_filters VALUES ('network-connectivity', 'Wired internet;Wireless internet;Cellular networks;Satellite internet;IoT networks;Fiber Optic Networks;Private Networks;Low-Power Wide-Area Network'); -INSERT INTO public.page_filters VALUES ('level-of-reliability', '1;2;3;4;5'); -INSERT INTO public.page_filters VALUES ('device-type', 'Desktop computers;Laptop computers;Tablets;Smartphones;GPS devices;Agricultural machinery equipped with IoT (Internet of Things) sensors'); --- Section 6 - Associated costs and prerequisites -INSERT INTO public.page_filters VALUES ('most-significant-costs', 'Initial investment;Connectivity infrastructure;Maintenance and upgrades;Energy;Integration with Existing Systems;Training and Skill Development;Data Security and Privacy Measures;Software development and privacy measures;Other'); -INSERT INTO public.page_filters VALUES ('direct-costs', 'High;Moderate;Low'); --- Section 7 - Data management and data sharing practices -INSERT INTO public.page_filters VALUES ('data-sharing', 'Open sharing;Restricted sharing;No sharing'); -INSERT INTO public.page_filters VALUES ('type-of-data', 'Crop and yield data;Soil data;Weather and environmental data;Pest and disease data;Inventory and equipment data;Market and economic data;Remote sensing and geospatial data;Livestock data;Financial and operational data'); -INSERT INTO public.page_filters VALUES ('types-of-tools-or-platforms', 'Field Data Collection Apps;Precision Agriculture Technology;IoT Devices;Remote Sensing Platforms;Farm Management Software;Forest Management Software;Forest Inventory Tools;Traceability Systems;Research Databases'); -INSERT INTO public.page_filters VALUES ('data-storage', 'On-premises servers/local storage facilities;Cloud-based platforms;Data warehouses;Agricultural information management systems;Geographic Information Systems (GIS);Hybrid storage solutions (on-premises and cloud);secire data centres (advanced security measures)'); --- Section 9 -- Socio-economic benefits and impact +INSERT INTO public.page_filters VALUES ('challenges-in-the-adoption', 'Yes;No;Don''t know'); +INSERT INTO public.page_filters VALUES ('barriers', 'Yes;No;Don''t know'); + +-- Section 4 - Technology Performance +INSERT INTO public.page_filters VALUES ('limitations-or-challenges', 'Yes;No;Don''t know'); +INSERT INTO public.page_filters VALUES ('network-connectivity', 'Yes;No;Don''t know'); + +-- Section 6 - Data Management and Data Sharing Practices +INSERT INTO public.page_filters VALUES ('data-collection', 'Yes;No;Don''t know'); +INSERT INTO public.page_filters VALUES ('type-of-data-collected', 'Crop and yield data;Soil data;Weather and environmental data;Pest and disease data;Inventory and equipment data;Market and economic data;Remote sensing and geospatial data;Livestock data;Financial and operational data'); +INSERT INTO public.page_filters VALUES ('data-payments', 'Yes;No;Don''t know'); +INSERT INTO public.page_filters VALUES ('data-sharing', 'Yes;No;Don''t know'); +INSERT INTO public.page_filters VALUES ('challenges-in-data-sharing', 'Yes;No;Don''t know'); INSERT INTO public.page_filters VALUES ('overall-contribution', 'Contribute;Impede;No impact'); -INSERT INTO public.page_filters VALUES ('savings-in-inputs', 'Significant;Moderate;Minimal'); -INSERT INTO public.page_filters VALUES ('job-creation-impact', 'Substantial impact;Moderate impact;Negligible impact'); -INSERT INTO public.page_filters VALUES ('overall-socio-economic-impact', 'Positive;Neutral;Negative'); --- Section 10 -- Environmental and sustainability impact + +-- Section 8 - Social Benefits and Impact +INSERT INTO public.page_filters VALUES ('social-benefits', 'Yes;No;Don''t know'); +INSERT INTO public.page_filters VALUES ('overall-social-impact', 'Positive;Neutral;Negative'); + +-- Section 9 - Economic Benefits and Impact +INSERT INTO public.page_filters VALUES ('cost-savings-efficiency', 'Yes;No;Don''t know'); +INSERT INTO public.page_filters VALUES ('savings-in-inputs', 'Yes;No;Don''t know'); +INSERT INTO public.page_filters VALUES ('overall-economic-impact', 'Positive;Neutral;Negative'); + +-- Section 10 - Environmental and Sustainability Impact INSERT INTO public.page_filters VALUES ('effects-on-biodiversity', 'Positive;Negative;No impact'); --- Section 11 -- Future outlook -INSERT INTO public.page_filters VALUES ('future-expansion/upgrade', 'Better connectivity/Infrastructure;More income/Access to funding;Standardisation efforts/Regulatory support;Better training and education'); \ No newline at end of file + +-- Section 11 - Future Outlook +INSERT INTO public.page_filters VALUES ('plans-to-expand-or-upgrade', 'Yes;No;Don''t know'); diff --git a/api/data/question-indicators.sql b/api/data/question-indicators.sql index cf5edf0f..25b73f7a 100644 --- a/api/data/question-indicators.sql +++ b/api/data/question-indicators.sql @@ -1,77 +1,116 @@ INSERT INTO question_indicator_map ("indicator", "question") VALUES - -- Not questions in the survey but widgets only - ('total-countries', ''), - ('total-surveys', ''), - ('adoption-of-technology-by-country', 'Has your organisation integrated digital technologies into its workflows?'), - -- Sections in the survey -- Section 1 - General Information - ('organisation-by-sector', 'Sector (agri/forestry/both)'), - ('eu-member-state', 'Location (country/region)'), + ('organisation-name', 'Organisation Name'), + ('sector', 'Sector (agri/forestry/both)'), ('type-of-stakeholder', 'Type of stakeholder'), - ('in-agriculture', 'Primary area of operation in agriculture'), - ('in-forestry', 'Primary area of operation in forestry'), - -- Section 2 - Workforce information + ('location-country-region', 'Location (country/region)'), + ('primary-area-of-operation-in-agriculture', 'Primary area of operation in agriculture'), + ('primary-area-of-operation-in-agriculture-other', 'If other, please specify'), + ('primary-area-of-operation-in-forestry', 'Primary area of operation in forestry'), + ('primary-area-of-operation-in-forestry-other', 'If other, please specify'), + ('organic-farming-operation', 'Organic farming operation'), ('organisation-size', 'Agriculture/forestry organisation size'), - ('education-level', 'What is the general education level/attainment of your workforce?'), - ('experience-level', 'What is the general level of work experience in your organisation?'), - ('training-on-digital-technologies', 'Have your staff received training on the use of digital technologies?'), - ('workforce-training', 'If yes, what percentage of the workforce received this training?'), - -- Section 3 - Governance model + ('regional-or-subsector-considerations', 'Are there specific regional or subsector considerations that should be taken into account when interpreting your responses?'), + ('considerations-to-be-taken-into-account', 'Considerations to be taken into account'), + + -- Section 2 - Governance Model ('governance-model-impact', 'Have specific governance models either facilitated or hindered the adoption of digital technologies in your organization?'), ('governance-model-type', 'What type of governance model do you operate under?'), + ('governance-model-type-other', 'If other, please specify'), ('regulatory-considerations', 'Are there regulatory considerations influencing the governance of digital technology adoption?'), - -- Section 4 - Adoption of Digital Technologies and Technology Integration + ('regulatory-considerations-details', 'If yes, please specify'), + + -- Section 3 - Adoption of Digital Technologies and Technology Integration ('digital-technologies-integrated', 'Has your organisation integrated digital technologies into its workflows?'), ('technology-type-agriculture', 'What type of digital technology has been used for agriculture?'), + ('technology-type-agriculture-other', 'If other, please specify'), ('technology-type-forestry', 'What type of digital technology has been used for forestry?'), + ('technology-type-forestry-other', 'If other, please specify'), ('goals-or-challenges', 'Were there specific goals or challenges that prompted the adoption of digital tools?'), + ('goals-or-challenges-details', 'If yes, please specify'), ('level-of-digitalization', 'How would you rate the level of digitalization in your farming/forestry practices on a scale of 1 to 5 (1 being low, 5 being high)'), ('primary-functions', 'What are the primary functions of these technologies in the agriculture or forestry value chain?'), ('adoption-level', 'What is the adoption level of these technologies?'), ('challenges-in-the-adoption', 'Have you encountered challenges in the adoption of digital technologies?'), + ('challenges-in-the-adoption-details', 'If yes, please specify'), ('barriers', 'Are there specific barriers hindering further integration?'), - -- Section 5 - Technology performance + ('barriers-details', 'If yes, please specify'), + + -- Section 4 - Technology Performance ('user-needs', 'To what extent do digital technologies meet evolving user needs within your organization?'), ('advantages', 'What are the advantages of the used technologies?'), + ('advantages-other', 'If other, please specify'), ('limitations-or-challenges', 'Have you encountered any perceived limitations or challenges in utilising these technologies?'), + ('limitations-or-challenges-details', 'If yes, please specify'), ('network-connectivity', 'Do you have network connectivity?'), ('network-connectivity-type', 'What network connectivity do you use?'), ('level-of-reliability', 'How reliable is the current network connectivity? (1 being not reliable, 5 being very reliable)'), + ('connectivity-barriers', 'Are there any specific barriers to accessing connectivity?'), ('device-type', 'What type of devices are commonly used to access the network?'), - -- Section 6 - Associated costs and prerequisites + + -- Section 5 - Associated Costs and Prerequisites ('most-significant-costs', 'What are the most significant costs associated with the adoption of digital technologies in your organisation'), + ('most-significant-costs-other', 'If other, please specify'), ('direct-costs', 'What is the level of direct costs?'), ('unexpected-or-hidden-costs', 'Unexpected or hidden costs?'), ('organisational-prerequisites', 'Have you identified organisational prerequisites (skills, workforce, education) necessary for successful technology integration?'), - -- Section 7 - Data management and data sharing practices + ('organisational-prerequisites-details', 'If yes, please specify'), + + -- Section 6 - Data Management and Data Sharing Practices ('data-collection', 'Is data collected from your farming/forestry activities?'), - ('data-sharing', 'What type of data sharing practices related to digital technology does your organisation use?'), - ('type-of-data', 'What type of data do you collect?'), + ('data-sharing-practices', 'What type of data sharing practices related to digital technology does your organisation use?'), + ('type-of-data-collected', 'What type of data do you collect?'), ('data-payments', 'Do you pay for this data?'), ('types-of-tools-or-platforms', 'What type of tools or platforms do you use to collect data?'), - ('data-sharing-2', 'Do you share this data?'), - ('challenges', 'Do challenges exist in sharing and interoperability of agricultural and forestry data?'), - ('%-data-driven-decisions', 'Approximately what percentage of overall decisions made are based on data analytics in your organisation?'), - -- Section 8 -- Data storage and data flows + ('data-sharing', 'Do you share this data?'), + ('challenges-in-data-sharing', 'Do challenges exist in sharing and interoperability of agricultural and forestry data?'), + ('challenges-in-data-sharing-details', 'If yes, please name the challenges'), + ('overall-contribution', 'How do these practices contribute to or impede the overall effectiveness of technology adoption?'), + ('percentage-data-driven-decisions', 'Approximately what percentage of overall decisions made are based on data analytics in your organisation?'), + + -- Section 7 - Data Storage and Data Flows ('data-storage', 'Where and how do you store this data?'), - ('data-flows/economic-implications', 'Are there economic implications associated with data flows in these sectors?'), + ('cloud-services', 'Do you use cloud services/data centres?'), + ('cloud-services-details', 'If yes, please name which cloud services/data centres'), + ('data-flows-economic-implications', 'Are there economic implications associated with data flows in these sectors?'), + ('data-flows-economic-implications-details', 'If yes, please name the main implications'), + ('data-flows-enhance-productivity', 'Do data flows enhance productivity and efficiency in agriculture and forestry?'), ('data-driven-decisions', 'Do you use data analytics for decision-making?'), - -- Section 9 -- Socio-economic benefits and impact - ('overall-contribution', 'How do these practices contribute to or impede the overall effectiveness of technology adoption?'), - ('socio-economic-benefits', 'Have you experienced socio-economic benefits through the use of digital technologies?'), - ('cost-savings/efficiency', 'Have digital technologies resulted in cost savings or increased efficiency?'), - ('savings-in-inputs', 'Savings in inputs?'), - ('overall-revenue', 'Increase in Overall revenue?'), - ('job-creation-impact', 'Impact on job creation?'), - ('overall-socio-economic-impact', 'What is the overall economic impact of implementing digital technologies?'), -- Also "Overall Socio-economic Impact?" - -- Section 10 -- Environmental and sustainability impact + ('data-receipt', 'Where do you receive data from and how much?'), + ('type-of-data-received-or-provided', 'What type of data do you receive or provide?'), + ('data-payments-received', 'Do you pay for this data?'), + ('data-payments-received-details', 'If yes, please specify'), + ('data-storage-received-data', 'Where and how do you store this data?'), + ('data-usage', 'What do you do with this data?'), + ('data-distribution', 'To whom and where do you send derived information or data?'), + + -- Section 8 - Social Benefits and Impact + ('social-benefits', 'Have you experienced social benefits through the use of digital technologies?'), + ('job-creation-impact', 'How have digital technologies impacted job creation?'), + ('overall-social-impact', 'What is the overall social impact of adopting digital technologies?'), + + -- Section 9 - Economic Benefits and Impact + ('cost-savings-efficiency', 'Have digital technologies resulted in cost savings or increased efficiency?'), + ('savings-in-inputs', 'Have you seen savings in inputs due to digital technologies?'), + ('overall-economic-impact', 'What is the overall economic impact of implementing digital technologies?'), + + -- Section 10 - Environmental and Sustainability Impact ('sustainability', 'Have digital technologies contributed to sustainability and environmental practices?'), ('conservation', 'Have you observed positive impacts on resource conservation or environmental footprint?'), ('energy-efficiency', 'Have digital technologies contributed to energy efficiency?'), ('effects-on-biodiversity', 'Have you observed any positive or negative effects on biodiversity in agricultural and forestry areas due to digital technology adoption?'), ('track-and-ensure-adherence', 'Do you use digital technologies to track and ensure adherence to sustainable farming practices and forestry activities?'), - -- Section 11 -- Future outlook - ('plans-to-expand/upgrade', 'Are there plans to expand or upgrade your current digital infrastructure?'), - ('future-expansion/upgrade', 'What would help facilitate the expansion/upgrade of digital infrastructure in the future?'), - ('future-developments', 'What type of developments do you anticipate in the near future?') -ON CONFLICT (indicator) DO UPDATE SET question = EXCLUDED.question; \ No newline at end of file + + -- Section 11 - Future Outlook + ('plans-to-expand-or-upgrade', 'Are there plans to expand or upgrade your current digital infrastructure?'), + ('future-expansion-upgrade', 'What would help facilitate the expansion/upgrade of digital infrastructure in the future?'), + ('future-developments', 'What type of developments do you anticipate in the near future?'), + + -- Section 12 - Additional Comments + ('additional-comments', 'Please share any other input that could be relevant to the questionnaire'), + + -- Not questions in the survey but widgets in the overview section + ('total-countries', ''), + ('total-surveys', ''), + ('adoption-of-technology-by-country', 'Has your organisation integrated digital technologies into its workflows?') +ON CONFLICT (indicator) DO UPDATE SET question = EXCLUDED.question; diff --git a/api/data/sections/sections.json b/api/data/sections/sections.json index 326b3e34..3fb2cbcd 100644 --- a/api/data/sections/sections.json +++ b/api/data/sections/sections.json @@ -28,7 +28,7 @@ }, { "question": "Sector (Agri/Forestry/Both)", - "indicator": "organisation-by-sector", + "indicator": "sector", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], "defaultVisualization": "horizontal_bar_chart", "sectionOrder": 4 @@ -36,79 +36,77 @@ ] }, { - "slug": "workforce-information", - "name": "Workforce information", - "description": "Workforce information", + "slug": "general-information", + "name": "General Information", + "description": "General Information", "order": 2, "baseWidgets": [ { - "question": "What is the size of your agriculture or forestry organisation in terms of workforce?", - "indicator": "organisation-size", + "question": "Type of stakeholder:", + "indicator": "type-of-stakeholder", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], "defaultVisualization": "horizontal_bar_chart", "sectionOrder": 1 }, { - "question": "What is the general education level/attainment of your workforce?", - "indicator": "education-level", - "visualisations": ["horizontal_bar_chart", "pie_chart"], - "defaultVisualization": "horizontal_bar_chart", + "question": "Location (Country/Region):", + "indicator": "location-country-region", + "visualisations": [ + "horizontal_bar_chart", + "pie_chart", + "area_graph", + "map" + ], + "defaultVisualization": "map", "sectionOrder": 2 }, { - "question": "What is the general level of work experience in your organisation?", - "indicator": "experience-level", - "visualisations": ["horizontal_bar_chart", "pie_chart"], + "question": "Primary Area of Operation in agriculture:", + "indicator": "primary-area-of-operation-in-agriculture", + "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], "defaultVisualization": "horizontal_bar_chart", "sectionOrder": 3 }, { - "question": "Have your staff received training on the use of digital technologies?", - "indicator": "training-on-digital-technologies", + "question": "Primary Area of Operation in forestry:", + "indicator": "primary-area-of-operation-in-forestry", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], "defaultVisualization": "horizontal_bar_chart", "sectionOrder": 4 }, { - "question": "If yes, what percentage of the workforce received this training?", - "indicator": "workforce-training", - "visualisations": ["horizontal_bar_chart", "pie_chart"], - "defaultVisualization": "horizontal_bar_chart", + "question": "Organic farming operation:", + "indicator": "organic-farming-operation", + "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], + "defaultVisualization": "pie_chart", "sectionOrder": 5 } ] }, { "slug": "governance-model", - "name": "Governance model", - "description": "Governance model", + "name": "Governance Model", + "description": "Governance Model", "order": 3, "baseWidgets": [ { "question": "Have specific governance models either facilitated or hindered the adoption of digital technologies in your organization?", "indicator": "governance-model-impact", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", + "defaultVisualization": "pie_chart", "sectionOrder": 1 }, - { - "question": "What type of governance model do you operate under?", - "indicator": "governance-model-type", - "visualisations": ["horizontal_bar_chart", "pie_chart"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 2 - }, { "question": "Are there regulatory considerations influencing the governance of digital technology adoption?", "indicator": "regulatory-considerations", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 3 + "defaultVisualization": "pie_chart", + "sectionOrder": 2 } ] }, { - "slug": "adoption-of-digital-technologies-and-technology-integration", + "slug": "adoption-of-digital-technologies", "name": "Adoption of Digital Technologies and Technology Integration", "description": "Adoption of Digital Technologies and Technology Integration", "order": 4, @@ -117,20 +115,20 @@ "question": "Has your organisation integrated digital technologies into its workflows?", "indicator": "digital-technologies-integrated", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", + "defaultVisualization": "pie_chart", "sectionOrder": 1 }, { "question": "What type of digital technology has been used for agriculture?", "indicator": "technology-type-agriculture", - "visualisations": ["horizontal_bar_chart", "pie_chart"], + "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], "defaultVisualization": "horizontal_bar_chart", "sectionOrder": 2 }, { "question": "What type of digital technology has been used for forestry?", "indicator": "technology-type-forestry", - "visualisations": ["horizontal_bar_chart", "pie_chart"], + "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], "defaultVisualization": "horizontal_bar_chart", "sectionOrder": 3 }, @@ -138,20 +136,25 @@ "question": "Were there specific goals or challenges that prompted the adoption of digital tools?", "indicator": "goals-or-challenges", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", + "defaultVisualization": "pie_chart", "sectionOrder": 4 }, { "question": "How would you rate the level of digitalization in your farming/forestry practices on a scale of 1 to 5 (1 being low, 5 being high)", "indicator": "level-of-digitalization", - "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], + "visualisations": [ + "horizontal_bar_chart", + "pie_chart", + "area_graph", + "gauge_chart" + ], "defaultVisualization": "horizontal_bar_chart", "sectionOrder": 5 }, { "question": "What are the primary functions of these technologies in the agriculture or forestry value chain?", "indicator": "primary-functions", - "visualisations": ["horizontal_bar_chart", "pie_chart"], + "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], "defaultVisualization": "horizontal_bar_chart", "sectionOrder": 6 }, @@ -159,21 +162,21 @@ "question": "What is the adoption level of these technologies?", "indicator": "adoption-level", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", + "defaultVisualization": "pie_chart", "sectionOrder": 7 }, { "question": "Have you encountered challenges in the adoption of digital technologies?", "indicator": "challenges-in-the-adoption", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", + "defaultVisualization": "pie_chart", "sectionOrder": 8 }, { "question": "Are there specific barriers hindering further integration?", "indicator": "barriers", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", + "defaultVisualization": "pie_chart", "sectionOrder": 9 } ] @@ -184,197 +187,120 @@ "description": "Technology Performance", "order": 5, "baseWidgets": [ - { - "question": "To what extent do digital technologies meet evolving user needs within your organization?", - "indicator": "user-needs", - "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 1 - }, - { - "question": "What are the advantages of the used technologies?", - "indicator": "advantages", - "visualisations": ["horizontal_bar_chart", "pie_chart"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 2 - }, { "question": "Have you encountered any perceived limitations or challenges in utilising these technologies?", "indicator": "limitations-or-challenges", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 3 + "defaultVisualization": "pie_chart", + "sectionOrder": 1 }, { "question": "Do you have network connectivity?", "indicator": "network-connectivity", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 4 - }, - { - "question": "What network connectivity do you use?", - "indicator": "network-connectivity-type", - "visualisations": ["horizontal_bar_chart", "pie_chart"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 5 - }, - { - "question": "How reliable is the current network connectivity?", - "indicator": "level-of-reliability", - "visualisations": ["horizontal_bar_chart", "pie_chart"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 6 - }, - { - "question": "What type of devices are commonly used to access the network?", - "indicator": "device-type", - "visualisations": ["horizontal_bar_chart", "pie_chart"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 7 - } - ] - }, - { - "slug": "associated-cost-and-prerequisites", - "name": "Associated cost and prerequisites", - "description": "Associated cost and prerequisites", - "order": 6, - "baseWidgets": [ - { - "question": "What are the most significant costs associated with the adoption of digital technologies in your organisation", - "indicator": "most-significant-costs", - "visualisations": ["horizontal_bar_chart", "pie_chart"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 1 - }, - { - "question": "What is the level of direct costs?", - "indicator": "direct-costs", - "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", + "defaultVisualization": "pie_chart", "sectionOrder": 2 - }, - { - "question": "Unexpected or hidden costs?", - "indicator": "unexpected-or-hidden-costs", - "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 3 - }, - { - "question": "Have you identified organisational prerequisites (skills, workforce, education) necessary for successful technology integration?", - "indicator": "organisational-prerequisites", - "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 4 } ] }, { - "slug": "data-management-and-data-sharing-practices", - "name": "Data management and data sharing practices", - "description": "Data management and data sharing practices", - "order": 7, + "slug": "data-management-and-sharing", + "name": "Data Management and Data Sharing Practices", + "description": "Data Management and Data Sharing Practices", + "order": 6, "baseWidgets": [ { "question": "Is data collected from your farming/forestry activities?", "indicator": "data-collection", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", + "defaultVisualization": "pie_chart", "sectionOrder": 1 }, { - "question": "What type of data sharing practices related to digital technology does your organisation use?", - "indicator": "data-sharing", + "question": "What type of data do you collect?", + "indicator": "type-of-data-collected", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], "defaultVisualization": "horizontal_bar_chart", "sectionOrder": 2 }, - { - "question": "What type of data do you collect?", - "indicator": "type-of-data", - "visualisations": ["horizontal_bar_chart", "pie_chart"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 3 - }, { "question": "Do you pay for this data?", "indicator": "data-payments", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 4 + "defaultVisualization": "pie_chart", + "sectionOrder": 3 }, { - "question": "What type of tools or platforms do you use to collect data?", - "indicator": "types-of-tools-or-platforms", - "visualisations": ["horizontal_bar_chart", "pie_chart"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 5 + "question": "Do you share this data?", + "indicator": "data-sharing", + "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], + "defaultVisualization": "pie_chart", + "sectionOrder": 4 }, { "question": "Do challenges exist in sharing and interoperability of agricultural and forestry data?", - "indicator": "challenges", + "indicator": "challenges-in-data-sharing", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 6 + "defaultVisualization": "pie_chart", + "sectionOrder": 5 }, { "question": "How do these practices contribute to or impede the overall effectiveness of technology adoption?", "indicator": "overall-contribution", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 7 + "sectionOrder": 6 } ] }, { - "slug": "socio-economic-benefits-and-impacts", - "name": "Socio-economic benefits and impacts", - "description": "Socio-economic benefits and impacts", - "order": 8, + "slug": "social-benefits-and-impact", + "name": "Social Benefits and Impact", + "description": "Social Benefits and Impact", + "order": 7, "baseWidgets": [ { - "question": "Have you experienced socio-economic benefits through the use of digital technologies?", - "indicator": "socio-economic-benefits", + "question": "Have you experienced social benefits through the use of digital technologies?", + "indicator": "social-benefits", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", + "defaultVisualization": "pie_chart", "sectionOrder": 1 }, { - "question": "Have digital technologies resulted in cost savings or increased efficiency?", - "indicator": "cost-savings/efficiency", + "question": "What is the overall social impact of adopting digital technologies?", + "indicator": "overall-social-impact", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], "defaultVisualization": "horizontal_bar_chart", "sectionOrder": 2 - }, - { - "question": "Savings in inputs?", - "indicator": "savings-in-inputs", - "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 3 - }, + } + ] + }, + { + "slug": "economic-benefits-and-impact", + "name": "Economic Benefits and Impact", + "description": "Economic Benefits and Impact", + "order": 8, + "baseWidgets": [ { - "question": "Increase in Overall revenue?", - "indicator": "overall-revenue", + "question": "Have digital technologies resulted in cost savings or increased efficiency?", + "indicator": "cost-savings-efficiency", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 4 + "defaultVisualization": "pie_chart", + "sectionOrder": 1 }, { - "question": "Impact on job creation?", - "indicator": "job-creation-impact", + "question": "Have you seen savings in inputs due to digital technologies?", + "indicator": "savings-in-inputs", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 5 + "defaultVisualization": "pie_chart", + "sectionOrder": 2 }, { - "question": "Overall Socio-economic Impact?", - "indicator": "overall-socio-economic-impact", + "question": "What is the overall economic impact of implementing digital technologies", + "indicator": "overall-economic-impact", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 6 + "sectionOrder": 3 } ] }, @@ -384,62 +310,27 @@ "description": "Environmental and Sustainability Impact", "order": 9, "baseWidgets": [ - { - "question": "Have digital technologies contributed to sustainability and environmental practices?", - "indicator": "sustainability", - "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 1 - }, - { - "question": "Have you observed positive impacts on resource conservation or environmental footprint?", - "indicator": "conservation", - "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 2 - }, - { - "question": "Have digital technologies contributed to energy efficiency?", - "indicator": "energy-efficiency", - "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 3 - }, { "question": "Have you observed any positive or negative effects on biodiversity in agricultural and forestry areas due to digital technology adoption?", "indicator": "effects-on-biodiversity", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 4 - }, - { - "question": "Do you use digital technologies to track and ensure adherence to sustainable farming practices and forestry activities?", - "indicator": "track-and-ensure-adherence", - "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 5 + "sectionOrder": 1 } ] }, { "slug": "future-outlook", - "name": "Future outlook", - "description": "Future outlook", + "name": "Future Outlook", + "description": "Future Outlook", "order": 10, "baseWidgets": [ { "question": "Are there plans to expand or upgrade your current digital infrastructure?", - "indicator": "plans-to-expand/upgrade", + "indicator": "plans-to-expand-or-upgrade", "visualisations": ["horizontal_bar_chart", "pie_chart", "area_graph"], - "defaultVisualization": "horizontal_bar_chart", + "defaultVisualization": "pie_chart", "sectionOrder": 1 - }, - { - "question": "What would help facilitate the expansion/upgrade of digital infrastructure in the future?", - "indicator": "future-expansion/upgrade", - "visualisations": ["horizontal_bar_chart", "pie_chart"], - "defaultVisualization": "horizontal_bar_chart", - "sectionOrder": 2 } ] } diff --git a/api/data/surveys/surveys.json b/api/data/surveys/surveys.json index 140a9037..6341ab9d 100644 --- a/api/data/surveys/surveys.json +++ b/api/data/surveys/surveys.json @@ -97,13 +97,6 @@ "answer": "No", "countryCode": "BEL" }, - { - "surveyId": "R_84qILa9AvSY5qCB", - "answerId": "248EA618033CE1ADF9F305AC20CF390BF3D0FB7E", - "question": "If you agree, please confirm the following statements i have read the information presented in this consent form...", - "answer": "Yes", - "countryCode": "BEL" - }, { "surveyId": "R_84qILa9AvSY5qCB", "answerId": "262A10478C2F5A61F261736F49D35C9D729E7B2A", @@ -314,13 +307,6 @@ "answer": "No", "countryCode": "BEL" }, - { - "surveyId": "R_84qILa9AvSY5qCB", - "answerId": "8C5419707E0981EC0FE4BDF524AF2A6961A97E50", - "question": "Specific regional or subsector considerations to take into account", - "answer": "Yes", - "countryCode": "BEL" - }, { "surveyId": "R_84qILa9AvSY5qCB", "answerId": "8EB4E2DC1F7472D415D118CF0F117CAACB1078C5", @@ -419,13 +405,6 @@ "answer": "Yes", "countryCode": "BEL" }, - { - "surveyId": "R_84qILa9AvSY5qCB", - "answerId": "D6480F0B50D39F79BFEB9572D599560BD9DD1535", - "question": "I agree to be contacted again by the researchers for clarification or elaboration on my input in the discussion (optional)", - "answer": "Yes", - "countryCode": "BEL" - }, { "surveyId": "R_84qILa9AvSY5qCB", "answerId": "D9C26DE72BF7B8262E6470FE4626CDF8085C557A", @@ -615,13 +594,6 @@ "answer": "No", "countryCode": "AUT" }, - { - "surveyId": "R_8OHmvlDNp8BXNAg", - "answerId": "5C95E525980BAABA64F88606F7347084D87C2D6B", - "question": "Specific regional or subsector considerations to take into account", - "answer": "Yes", - "countryCode": "AUT" - }, { "surveyId": "R_8OHmvlDNp8BXNAg", "answerId": "5E554E9B6E1C982A099618148078682F03FF38D5", @@ -685,13 +657,6 @@ "answer": "On-premises servers/local storage facilities", "countryCode": "AUT" }, - { - "surveyId": "R_8OHmvlDNp8BXNAg", - "answerId": "A552CFB592058E51843E7AB0C3757503FDF65C14", - "question": "If you agree, please confirm the following statements i have read the information presented in this consent form...", - "answer": "Yes", - "countryCode": "AUT" - }, { "surveyId": "R_8OHmvlDNp8BXNAg", "answerId": "A8075694F4C09A8A0E67F5E286A0602C16839EFC", @@ -748,13 +713,6 @@ "answer": "No", "countryCode": "AUT" }, - { - "surveyId": "R_8OHmvlDNp8BXNAg", - "answerId": "CFB12DAB0E2C261C35A46E29C741569F9AC75C55", - "question": "I agree to be contacted again by the researchers for clarification or elaboration on my input in the discussion (optional)", - "answer": "No", - "countryCode": "AUT" - }, { "surveyId": "R_8OHmvlDNp8BXNAg", "answerId": "D3AEDD974FE3181B34AC4ABE6F55212F71B939DF", @@ -853,13 +811,6 @@ "answer": "Both", "countryCode": "BGR" }, - { - "surveyId": "R_2eKIDqs0Vsyj0qM", - "answerId": "773B2CC3D69DB30DCCC22E302250D265516789C9", - "question": "I agree to be contacted again by the researchers for clarification or elaboration on my input in the discussion (optional)", - "answer": "No", - "countryCode": "BGR" - }, { "surveyId": "R_2eKIDqs0Vsyj0qM", "answerId": "79A70081AB594B4BE815007FA398C1471B68E143", @@ -867,13 +818,6 @@ "answer": "Medium-scale/Local-National", "countryCode": "BGR" }, - { - "surveyId": "R_2eKIDqs0Vsyj0qM", - "answerId": "A9160B10D15EECF1E9033198D32E0B548A190ED0", - "question": "Specific regional or subsector considerations to take into account", - "answer": "Yes", - "countryCode": "BGR" - }, { "surveyId": "R_2eKIDqs0Vsyj0qM", "answerId": "B08C923EA4C83BDD125F723D59965F678E156C7A", @@ -881,13 +825,6 @@ "answer": "Bulgaria", "countryCode": "BGR" }, - { - "surveyId": "R_2eKIDqs0Vsyj0qM", - "answerId": "D33FF6FF8979C50CEAFA57EFD9BD9E0484F118BF", - "question": "If you agree, please confirm the following statements i have read the information presented in this consent form...", - "answer": "Yes", - "countryCode": "BGR" - }, { "surveyId": "R_2hbsGDfZSDyw8BX", "answerId": "1AF61EAD18A56CC7B62F14857B94B3B9060B217C", @@ -902,13 +839,6 @@ "answer": "Don't know", "countryCode": "NLD" }, - { - "surveyId": "R_2hbsGDfZSDyw8BX", - "answerId": "1F32771828317F048FA73788B471DC363C159386", - "question": "If you agree, please confirm the following statements i have read the information presented in this consent form...", - "answer": "Yes", - "countryCode": "NLD" - }, { "surveyId": "R_2hbsGDfZSDyw8BX", "answerId": "2055C5821CC3CEB8FA6328DC5F4A4D5A0DC33B7D", @@ -1000,20 +930,6 @@ "answer": "Netherlands", "countryCode": "NLD" }, - { - "surveyId": "R_2hbsGDfZSDyw8BX", - "answerId": "935D0F300ED2BC92B5A1DE061AA77CA7E74C49BB", - "question": "Specific regional or subsector considerations to take into account", - "answer": "No", - "countryCode": "NLD" - }, - { - "surveyId": "R_2hbsGDfZSDyw8BX", - "answerId": "A369DBD4098AA9EEB2BEB0F11146AE7B04568F22", - "question": "I agree to be contacted again by the researchers for clarification or elaboration on my input in the discussion (optional)", - "answer": "Yes", - "countryCode": "NLD" - }, { "surveyId": "R_2hbsGDfZSDyw8BX", "answerId": "A4081F1DCEE4F3A9EBB99C7720F1A171FC9A60C2", @@ -1084,20 +1000,6 @@ "answer": "Efficient resource allocation", "countryCode": "BEL" }, - { - "surveyId": "R_2XckmVyZ44klhUe", - "answerId": "7F4AEA6D791AA42EF14B9A1CA5D2A9E240F07A48", - "question": "I agree to be contacted again by the researchers for clarification or elaboration on my input in the discussion (optional)", - "answer": "Yes", - "countryCode": "BEL" - }, - { - "surveyId": "R_2XckmVyZ44klhUe", - "answerId": "938ECB9906C797C36BA012784BCF07C60D370C73", - "question": "If you agree, please confirm the following statements i have read the information presented in this consent form...", - "answer": "Yes", - "countryCode": "BEL" - }, { "surveyId": "R_2XckmVyZ44klhUe", "answerId": "B4A5695CAF35D3787982334EF22CFDE48C180AFE", diff --git a/api/data/surveys/transform.ts b/api/data/surveys/transform.ts index 100538c4..6741be1e 100644 --- a/api/data/surveys/transform.ts +++ b/api/data/surveys/transform.ts @@ -1,8 +1,16 @@ import * as dataForge from 'data-forge'; -import * as fs from 'node:fs'; +import * as fs from 'fs'; import { CountryISO3Map } from '@shared/constants/country-iso3.map'; import { StringUtils } from '@api/utils/string.utils'; +const EXCLUDED_QUESTIONS = { + 'If you agree, please confirm the following statements i have read the information presented in this consent form...': + true, + 'Specific regional or subsector considerations to take into account': true, + 'I agree to be contacted again by the researchers for clarification or elaboration on my input in the discussion (optional)': + true, +}; + const main = async () => { const [answersDf, dateDf, questionDf, answerIdDf] = await Promise.all([ dataForge.fromObject( @@ -68,7 +76,7 @@ const main = async () => { (row) => row['surveyId'], ); - const transformedAnswers = []; + let transformedAnswers = []; for (const group of answersGroupByAnswers) { let answers = group.toArray(); const countryAnswer = answers.find( @@ -90,6 +98,9 @@ const main = async () => { transformedAnswers.push(...answers); } + transformedAnswers = transformedAnswers.filter( + (e) => !EXCLUDED_QUESTIONS[e.question], + ); fs.writeFileSync( `${__dirname}/surveys.json`, JSON.stringify(transformedAnswers, null, 2), diff --git a/api/src/infrastructure/data-source-manager.ts b/api/src/infrastructure/data-source-manager.ts index 79863363..1bd65066 100644 --- a/api/src/infrastructure/data-source-manager.ts +++ b/api/src/infrastructure/data-source-manager.ts @@ -1,8 +1,7 @@ import * as fs from 'fs'; -import { Inject, Injectable, Logger } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { InjectDataSource } from '@nestjs/typeorm'; import { DataSource } from 'typeorm'; -import { SQLAdapter } from '@api/infrastructure/sql-adapter'; import { SectionsJSONParser } from 'api/data/sections/sections-json-parser'; import { SurveyAnswer } from '@shared/dto/surveys/survey-answer.entity'; import { QuestionIndicatorMap } from '@shared/dto/surveys/question-widget-map.entity'; @@ -12,7 +11,6 @@ import { Section } from '@shared/dto/sections/section.entity'; export class DataSourceManager { public constructor( private readonly logger: Logger, - @Inject(SQLAdapter) private readonly sqlAdapter: SQLAdapter, @InjectDataSource() private readonly dataSource: DataSource, ) {} diff --git a/api/src/infrastructure/postgres-survey-data.repository.ts b/api/src/infrastructure/postgres-survey-data.repository.ts index 8e5a39fb..0889467d 100644 --- a/api/src/infrastructure/postgres-survey-data.repository.ts +++ b/api/src/infrastructure/postgres-survey-data.repository.ts @@ -13,6 +13,8 @@ import { import { WidgetUtils } from '@shared/dto/widgets/widget.utils'; export class PostgresSurveyDataRepository implements ISurveyDataRepository { + private readonly answersTable: string = 'survey_answers'; + public constructor( private readonly logger: Logger, @Inject(SQLAdapter) private readonly sqlAdapter: SQLAdapter, @@ -23,9 +25,7 @@ export class PostgresSurveyDataRepository implements ISurveyDataRepository { sections: SectionWithDataWidget[], filters?: WidgetDataFilters, ): Promise { - const answersTable: string = 'survey_answers'; let filterClause: string; - if (filters !== undefined) { filterClause = this.sqlAdapter.generateSqlFromWidgetDataFilters(filters); } @@ -37,7 +37,7 @@ export class PostgresSurveyDataRepository implements ISurveyDataRepository { for (let widgetIdx = 0; widgetIdx < baseWidgets.length; widgetIdx++) { const widget = baseWidgets[widgetIdx]; widgetDataPromises.push( - this.appendBaseWidgetData(answersTable, filterClause, widget), + this.appendBaseWidgetData(widget, filterClause), ); } } @@ -46,16 +46,28 @@ export class PostgresSurveyDataRepository implements ISurveyDataRepository { return sections; } + public async addSurveyDataToBaseWidget( + widget: BaseWidgetWithData, + filters?: WidgetDataFilters, + ): Promise { + let filterClause: string; + if (filters !== undefined) { + filterClause = this.sqlAdapter.generateSqlFromWidgetDataFilters(filters); + } + + await this.appendBaseWidgetData(widget, filterClause); + return widget; + } + private async appendBaseWidgetData( - answersTable: string, - filterClause: string, widget: BaseWidgetWithData, + filterClause: string, ): Promise { const { indicator } = widget; // Edge cases here. Total surveys and total countries. if (indicator === 'total-surveys') { - const filteredCount = `SELECT COUNT(count)::integer as count FROM (SELECT COUNT(DISTINCT survey_id) FROM ${answersTable} ${filterClause} GROUP BY survey_id) AS survey_count`; - const totalCount = `SELECT COUNT(count)::integer as count FROM (SELECT COUNT(DISTINCT survey_id) FROM ${answersTable} GROUP BY survey_id) AS survey_count`; + const filteredCount = `SELECT COUNT(count)::integer as count FROM (SELECT COUNT(DISTINCT survey_id) FROM ${this.answersTable} ${filterClause} GROUP BY survey_id) AS survey_count`; + const totalCount = `SELECT COUNT(count)::integer as count FROM (SELECT COUNT(DISTINCT survey_id) FROM ${this.answersTable} GROUP BY survey_id) AS survey_count`; const [[{ count: value }], [{ count: total }]] = await Promise.all([ this.dataSource.query(filteredCount), this.dataSource.query(totalCount), @@ -64,8 +76,8 @@ export class PostgresSurveyDataRepository implements ISurveyDataRepository { return; } if (indicator === 'total-countries') { - const filteredCount = `SELECT COUNT(DISTINCT country_code)::integer as "count" FROM ${answersTable} ${filterClause}`; - const totalCount = `SELECT COUNT(DISTINCT country_code)::integer as "count" FROM ${answersTable};`; + const filteredCount = `SELECT COUNT(DISTINCT country_code)::integer as "count" FROM ${this.answersTable} ${filterClause}`; + const totalCount = `SELECT COUNT(DISTINCT country_code)::integer as "count" FROM ${this.answersTable};`; const [[{ count: value }], [{ count: total }]] = await Promise.all([ this.dataSource.query(filteredCount), this.dataSource.query(totalCount), @@ -74,7 +86,7 @@ export class PostgresSurveyDataRepository implements ISurveyDataRepository { return; } - const totalsSql = `SELECT answer as "key", count(answer)::integer as "count", SUM(COUNT(answer)) OVER ()::integer AS total FROM ${answersTable} ${this.sqlAdapter.appendExpressionToFilterClause(filterClause, `question_indicator = '${indicator}'`)} GROUP BY answer`; + const totalsSql = `SELECT answer as "key", count(answer)::integer as "count", SUM(COUNT(answer)) OVER ()::integer AS total FROM ${this.answersTable} ${this.sqlAdapter.appendExpressionToFilterClause(filterClause, `question_indicator = '${indicator}'`)} GROUP BY answer`; const totalsResult: { key: string; count: number; total: number }[] = await this.dataSource.query(totalsSql); diff --git a/api/src/infrastructure/survey-data-repository.interface.ts b/api/src/infrastructure/survey-data-repository.interface.ts index 7f67481b..1f0a30b6 100644 --- a/api/src/infrastructure/survey-data-repository.interface.ts +++ b/api/src/infrastructure/survey-data-repository.interface.ts @@ -1,8 +1,20 @@ -import { Section } from '@shared/dto/sections/section.entity'; +import { + Section, + SectionWithDataWidget, +} from '@shared/dto/sections/section.entity'; +import { BaseWidgetWithData } from '@shared/dto/widgets/base-widget-data.interface'; +import { BaseWidget } from '@shared/dto/widgets/base-widget.entity'; import { WidgetDataFilter } from '@shared/dto/widgets/widget-data-filter'; + +export const SurveyDataRepository = Symbol('ISurveyDataRepository'); + export interface ISurveyDataRepository { addSurveyDataToSections( sections: Partial
[], filters?: WidgetDataFilter[], - ): Promise; + ): Promise; + addSurveyDataToBaseWidget( + widget: BaseWidget, + filters?: WidgetDataFilter[], + ): Promise; } diff --git a/api/src/modules/sections/sections.module.ts b/api/src/modules/sections/sections.module.ts index 4e52c5e4..f96a407a 100644 --- a/api/src/modules/sections/sections.module.ts +++ b/api/src/modules/sections/sections.module.ts @@ -6,12 +6,13 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { AuthModule } from '@api/modules/auth/auth.module'; import { PostgresSurveyDataRepository } from '@api/infrastructure/postgres-survey-data.repository'; import { SQLAdapter } from '@api/infrastructure/sql-adapter'; +import { SurveyDataRepository } from '@api/infrastructure/survey-data-repository.interface'; @Module({ imports: [TypeOrmModule.forFeature([Section]), forwardRef(() => AuthModule)], providers: [ SQLAdapter, - { provide: 'SurveyDataRepository', useClass: PostgresSurveyDataRepository }, + { provide: SurveyDataRepository, useClass: PostgresSurveyDataRepository }, SectionsService, ], controllers: [SectionsController], diff --git a/api/src/modules/sections/sections.service.ts b/api/src/modules/sections/sections.service.ts index 8a6b9159..ee9f49dc 100644 --- a/api/src/modules/sections/sections.service.ts +++ b/api/src/modules/sections/sections.service.ts @@ -8,7 +8,10 @@ import { SectionWithDataWidget, } from '@shared/dto/sections/section.entity'; import { FetchSpecification } from 'nestjs-base-service'; -import { ISurveyDataRepository } from '@api/infrastructure/survey-data-repository.interface'; +import { + ISurveyDataRepository, + SurveyDataRepository, +} from '@api/infrastructure/survey-data-repository.interface'; import { WidgetDataFiltersSchema } from '@shared/schemas/widget-data-filters.schema'; @Injectable() @@ -21,7 +24,7 @@ export class SectionsService extends AppBaseService< public constructor( @InjectRepository(Section) private readonly sectionsRepository: Repository, - @Inject('SurveyDataRepository') + @Inject(SurveyDataRepository) private readonly surveyDataRepository: ISurveyDataRepository, ) { super(sectionsRepository, 'section', 'sections'); diff --git a/api/src/modules/widgets/widgets.controller.ts b/api/src/modules/widgets/widgets.controller.ts index 3f966224..41486879 100644 --- a/api/src/modules/widgets/widgets.controller.ts +++ b/api/src/modules/widgets/widgets.controller.ts @@ -22,7 +22,10 @@ export class WidgetsController { @TsRestHandler(c.getWidget) async getWidget(): Promise { return tsRestHandler(c.getWidget, async ({ params: { id }, query }) => { - const widget = await this.widgetsService.getById(id, query); + const widget = await this.widgetsService.findWidgetWithDataById( + id, + query, + ); return { body: { data: widget }, status: HttpStatus.OK }; }); } diff --git a/api/src/modules/widgets/widgets.module.ts b/api/src/modules/widgets/widgets.module.ts index def04b61..aa915b08 100644 --- a/api/src/modules/widgets/widgets.module.ts +++ b/api/src/modules/widgets/widgets.module.ts @@ -10,13 +10,22 @@ import { PageFiltersController } from '@api/modules/widgets/page-filters.control import { PageFilter } from '@shared/dto/widgets/page-filter.entity'; import { WidgetsController } from '@api/modules/widgets/widgets.controller'; import { WidgetsService } from '@api/modules/widgets/widgets.service'; +import { PostgresSurveyDataRepository } from '@api/infrastructure/postgres-survey-data.repository'; +import { SQLAdapter } from '@api/infrastructure/sql-adapter'; +import { SurveyDataRepository } from '@api/infrastructure/survey-data-repository.interface'; @Module({ imports: [ TypeOrmModule.forFeature([BaseWidget, CustomWidget, PageFilter]), forwardRef(() => AuthModule), ], - providers: [CustomWidgetService, PageFiltersService, WidgetsService], + providers: [ + SQLAdapter, + { provide: SurveyDataRepository, useClass: PostgresSurveyDataRepository }, + CustomWidgetService, + PageFiltersService, + WidgetsService, + ], controllers: [ CustomWidgetsController, PageFiltersController, diff --git a/api/src/modules/widgets/widgets.service.ts b/api/src/modules/widgets/widgets.service.ts index 5af1b5a6..424a88e1 100644 --- a/api/src/modules/widgets/widgets.service.ts +++ b/api/src/modules/widgets/widgets.service.ts @@ -1,11 +1,17 @@ import { AppBaseService } from '@api/utils/app-base.service'; import { AppInfoDTO } from '@api/utils/info.dto'; -import { Injectable, Logger } from '@nestjs/common'; +import { Inject, Injectable, Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { BaseWidget } from '@shared/dto/widgets/base-widget.entity'; import { Repository, SelectQueryBuilder } from 'typeorm'; import { FetchSpecification } from 'nestjs-base-service'; import { WidgetVisualisationFilters } from '@shared/schemas/widget-visualisation-filters.schema'; +import { BaseWidgetWithData } from '@shared/dto/widgets/base-widget-data.interface'; +import { WidgetDataFiltersSchema } from '@shared/schemas/widget-data-filters.schema'; +import { + ISurveyDataRepository, + SurveyDataRepository, +} from '@api/infrastructure/survey-data-repository.interface'; @Injectable() export class WidgetsService extends AppBaseService< @@ -19,8 +25,10 @@ export class WidgetsService extends AppBaseService< protected readonly logger: Logger, @InjectRepository(BaseWidget) private baseWidgetRepository: Repository, + @Inject(SurveyDataRepository) + private readonly surveyDataRepository: ISurveyDataRepository, ) { - super(baseWidgetRepository, 'baseWidget', 'basedWidgets'); + super(baseWidgetRepository, 'baseWidget', 'baseWidgets'); } async extendFindAllQuery( @@ -46,4 +54,20 @@ export class WidgetsService extends AppBaseService< ); return query; } + + public async findWidgetWithDataById( + id: string, + query: FetchSpecification & WidgetDataFiltersSchema, + ): Promise { + const widget = await this.baseWidgetRepository.findOneBy({ indicator: id }); + const { filters } = query; + + const baseWidgetWithData = + await this.surveyDataRepository.addSurveyDataToBaseWidget( + widget, + filters, + ); + + return baseWidgetWithData; + } } diff --git a/api/test/e2e/sections/sections-crud.spec.ts b/api/test/e2e/sections/sections-crud.spec.ts index 9ec1ab99..1a4b05a2 100644 --- a/api/test/e2e/sections/sections-crud.spec.ts +++ b/api/test/e2e/sections/sections-crud.spec.ts @@ -46,7 +46,7 @@ describe('Page Sections API', () => { }), await entityMocks.createBaseWidget({ sectionOrder: 2, - indicator: 'organisation-by-sector', + indicator: 'sector', }), ], }); @@ -55,7 +55,7 @@ describe('Page Sections API', () => { const res = await testManager .request() .get( - '/sections?filters[0][name]=eu-member-state&filters[0][operator]==&filters[0][values]=Spain&filters[0][values]=Belgium', + '/sections?filters[0][name]=eu-member-state&filters[0][operator]==&filters[0][values][0]=Belgium', ); // Implicit ?include[]=baseWidgets&sort[]=order&sort[]=baseWidget.sectionOrder // Then @@ -71,7 +71,7 @@ describe('Page Sections API', () => { const expectedWidgetData = { 'total-countries': { counter: { value: 1, total: 4 } }, 'total-surveys': { counter: { value: 2, total: 5 } }, - 'organisation-by-sector': { + sector: { chart: [ { label: 'Agriculture', diff --git a/api/test/e2e/widgets/base-widgets.spec.ts b/api/test/e2e/widgets/base-widgets.spec.ts index 8048413e..71640d48 100644 --- a/api/test/e2e/widgets/base-widgets.spec.ts +++ b/api/test/e2e/widgets/base-widgets.spec.ts @@ -2,6 +2,7 @@ import { TestManager } from 'api/test/utils/test-manager'; import { WidgetVisualizationsType } from '@shared/dto/widgets/widget-visualizations.constants'; import { DataSourceManager } from '@api/infrastructure/data-source-manager'; import { BaseWidget } from '@shared/dto/widgets/base-widget.entity'; +import ObjectUtils from 'api/test/utils/object.utils'; describe('Base Widgets', () => { let testManager: TestManager; @@ -56,11 +57,6 @@ describe('Base Widgets', () => { const mocks = testManager.mocks(); for (const [i, vizz] of vizzes.entries()) { const indicator = `indicator-${i}`; - // To make sure we satify the FK_constraint - await mocks.createQuestionIndicatorMap({ - indicator, - question: indicator, - }); await mocks.createBaseWidget({ indicator, visualisations: vizz as WidgetVisualizationsType[], @@ -94,4 +90,65 @@ describe('Base Widgets', () => { expect(noneFound.body.data).toHaveLength(0); }); + + it('Should retrieve a widget with its data by its id (indicator)', async () => { + // Given + const dataSourceManager = testManager.testApp.get(DataSourceManager); + await dataSourceManager.loadQuestionIndicatorMap(); + await dataSourceManager.loadSurveyData( + `${__dirname}/../../data/surveys.json`, + ); + + const indicator = 'total-surveys'; + const mocks = testManager.mocks(); + const createdWidget = await mocks.createBaseWidget({ + indicator, + }); + + // When + const result = await testManager.request().get(`/widgets/${indicator}`); + const returnedWidget = result.body.data; + + const createdWidgetWithData = { + ...ObjectUtils.normalizeDates(createdWidget), + data: { + counter: { value: 5, total: 5 }, + }, + }; + // Then + expect(returnedWidget).toStrictEqual(createdWidgetWithData); + }); + + it('Should retrieve a widget with its filtered data by its id (indicator)', async () => { + // Given + const dataSourceManager = testManager.testApp.get(DataSourceManager); + await dataSourceManager.loadQuestionIndicatorMap(); + await dataSourceManager.loadSurveyData( + `${__dirname}/../../data/surveys.json`, + ); + + const indicator = 'total-surveys'; + const mocks = testManager.mocks(); + const createdWidget = await mocks.createBaseWidget({ + indicator, + }); + + // When + const result = await testManager + .request() + .get( + `/widgets/${indicator}?filters[0][name]=eu-member-state&filters[0][operator]==&filters[0][values][0]=Belgium`, + ); + + const returnedWidget = result.body.data; + + const createdWidgetWithData = { + ...ObjectUtils.normalizeDates(createdWidget), + data: { + counter: { value: 2, total: 5 }, + }, + }; + // Then + expect(returnedWidget).toStrictEqual(createdWidgetWithData); + }); }); diff --git a/api/test/utils/test-manager.ts b/api/test/utils/test-manager.ts index 1f29098c..bb9b2e53 100644 --- a/api/test/utils/test-manager.ts +++ b/api/test/utils/test-manager.ts @@ -9,6 +9,7 @@ import { createQuestionIndicatorMap, createSection, createUser, + ensureQuestionIndicatorMapExists, } from '@shared/lib/entity-mocks'; import { logUserIn } from './user.auth'; import { Type } from '@nestjs/common/interfaces'; @@ -107,10 +108,13 @@ export class TestManager { createQuestionIndicatorMap: async ( additionalData: QuestionIndicatorMap, ) => createQuestionIndicatorMap(this.getDataSource(), additionalData), + ensureQuestionIndicatorMapExists: async ( + dataSource: DataSource, + questionIndicatorMap: { indicator: string; question: string }, + ) => ensureQuestionIndicatorMapExists(dataSource, questionIndicatorMap), createBaseWidget: ( - additionalData?: DeepPartial, - ): Promise => - createBaseWidget(this.getDataSource(), additionalData), + data: DeepPartial & { indicator: string }, + ): Promise => createBaseWidget(this.getDataSource(), data), createSection: (additionalData: DeepPartial
) => createSection(this.getDataSource(), additionalData), createCustomWidget: ( diff --git a/shared/contracts/base-widgets.contract.ts b/shared/contracts/base-widgets.contract.ts index a2491a2f..4c54aeba 100644 --- a/shared/contracts/base-widgets.contract.ts +++ b/shared/contracts/base-widgets.contract.ts @@ -8,6 +8,7 @@ import { import { generateEntityQuerySchema } from '@shared/schemas/query-param.schema'; import { BaseWidget } from '@shared/dto/widgets/base-widget.entity'; import { WidgetVisualizationFilterSchema } from '@shared/schemas/widget-visualisation-filters.schema'; +import { WidgetDataFiltersSchema } from '@shared/schemas/widget-data-filters.schema'; const contract = initContract(); export const widgetsContract = contract.router({ @@ -29,9 +30,7 @@ export const widgetsContract = contract.router({ pathParams: z.object({ id: z.coerce.string(), }), - query: generateEntityQuerySchema(BaseWidget).merge( - WidgetVisualizationFilterSchema, - ), + query: generateEntityQuerySchema(BaseWidget).merge(WidgetDataFiltersSchema), responses: { 200: contract.type>(), 400: contract.type(), diff --git a/shared/lib/e2e-test-manager.ts b/shared/lib/e2e-test-manager.ts index 08369ae2..445d9fa1 100644 --- a/shared/lib/e2e-test-manager.ts +++ b/shared/lib/e2e-test-manager.ts @@ -5,6 +5,7 @@ import { createCustomWidget, createSection, createUser, + ensureQuestionIndicatorMapExists, } from '@shared/lib/entity-mocks'; import { clearTestDataFromDatabase } from '@shared/lib/db-helpers'; import { DB_ENTITIES } from '@shared/lib/db-entities'; @@ -64,13 +65,16 @@ export class E2eTestManager { createSection: (additionalData?: DeepPartial
) => createSection(this.getDataSource(), additionalData), createBaseWidget: ( - additionalData?: DeepPartial, - ): Promise => - createBaseWidget(this.getDataSource(), additionalData), + data: DeepPartial & { indicator: string }, + ): Promise => createBaseWidget(this.getDataSource(), data), createCustomWidget: ( additionalData?: DeepPartial, ): Promise => createCustomWidget(this.getDataSource(), additionalData), + ensureQuestionIndicatorMapExists: async ( + dataSource: DataSource, + questionIndicatorMap: { indicator: string; question: string }, + ) => ensureQuestionIndicatorMapExists(dataSource, questionIndicatorMap), }; } diff --git a/shared/lib/entity-mocks.ts b/shared/lib/entity-mocks.ts index cfea762a..64e756e4 100644 --- a/shared/lib/entity-mocks.ts +++ b/shared/lib/entity-mocks.ts @@ -32,11 +32,31 @@ export const createQuestionIndicatorMap = async ( return repo.save(data); }; +export const ensureQuestionIndicatorMapExists = async ( + dataSource: DataSource, + questionIndicatorMap: { indicator: string; question: string }, +) => { + const questionIndicatorRepository = + dataSource.getRepository(QuestionIndicatorMap); + const foundQuestionIndicator = await questionIndicatorRepository.findOneBy({ + indicator: questionIndicatorMap.indicator, + }); + if (foundQuestionIndicator === null) { + return await questionIndicatorRepository.save(questionIndicatorMap); + } + return undefined; +}; + export const createBaseWidget = async ( dataSource: DataSource, - additionalData?: DeepPartial, + data: DeepPartial & { indicator: string }, ) => { - // TODO: Create QuestionMapIndicator if not exists to reduce the verbosity of the test files + const indicator = data.indicator; + + await ensureQuestionIndicatorMapExists(dataSource, { + indicator, + question: indicator, + }); const baseWidgetsRepository = dataSource.getRepository(BaseWidget); const defaults: Partial = { @@ -48,7 +68,7 @@ export const createBaseWidget = async ( sectionOrder: 1, }; - return baseWidgetsRepository.save({ ...defaults, ...additionalData }); + return baseWidgetsRepository.save({ ...defaults, ...data }); }; export const createSection = async ( @@ -64,6 +84,20 @@ export const createSection = async ( order: 1, }; + // Create indicators if they don't exist in the database + if (additionalData !== undefined) { + const baseWidgets = additionalData.baseWidgets; + if (Array.isArray(baseWidgets) === true) { + for (let widgetIdx = 0; widgetIdx < baseWidgets.length; widgetIdx++) { + const widget = baseWidgets[widgetIdx]; + await ensureQuestionIndicatorMapExists(dataSource, { + indicator: widget.indicator as string, + question: widget.indicator as string, + }); + } + } + } + return sectionsRepository.save({ ...defaults, ...additionalData }); };