R/fpds.R
In abresler/govtrackR: Federal government tracking tools
Defines functions
.read_fdps_csv
.create_fpds_query_params
.munge_double_list
.create_date_slug
.count_results
.shorten_character
.shorten_count
.shorten_amounts
.shorten_dates
.create_value_slug
.sort_slug
.munge_fpds_names
.munge_fdps_data
pad_fdps_duns
.pad_fdps_duns
dictionary_fpds_search_parameters
dictionary_fpds_parameter_description
Documented in
dictionary_fpds_parameter_description
dictionary_fpds_search_parameters
pad_fdps_duns
# https://www.fpds.gov/wiki/index.php/FPDS-NG_FAQ
# https://www.fpds.gov/wiki/index.php/ATOM_Feed_FAQ
#' FPDS parameter description dictionary
#'
#' @return
#' @export
#'
#' @examples
dictionary_fpds_parameter_description <- function() {
page <-
"https://www.fpds.gov/wiki/index.php/EzSearch_FAQ" %>% read_html()
tables <- page %>% html_table(fill = F)
data <- tables[[1]] %>% as_tibble()
data <-
data %>% setNames(c(
"nameItem",
"parameterSearch",
"typeSearch",
"descriptionSearch"
))
data
}
# dictionaries ------------------------------------------------------------
#' FPDS Search Parameter Dictionary
#'
#' @return
#' @export
#'
#' @examples
#' dictionary_fpds_search_parameters()
dictionary_fpds_search_parameters <-
function() {
parameter <-
c(
"CONTRACT_STATUS_DESCRIPTION",
"VENDOR_SITE_CODE_ALTERNATE",
"PROGRAM_ACRONYM",
"AD_POST_AWARD_AUDIT",
"CURRENT_CONTRACT_VALUE",
"CONTRACT_TYPE_DESCRIPTION",
"AD_NONE",
"CONTRACTOR_NAME",
"CONTRACT_FINANCING_DESC",
"SECURITY_CODE",
"TRANSACTION_NUMBER",
"NGC_PARTICIPATION_DESC",
"REASON_NOT_COMPETED",
"RECOVERED_MATERIAL_CLS_DESC",
"ORIGINAL_PIID",
"PRINCIPAL_INV_FIRST_NAME",
"EMAIL_ADDRESS",
"PRODUCT_OR_SERVICE_CODE",
"LEGISLATIVE_MANDATES",
"SERVICE_CONTRACT_ACT",
"WALSH_HEALEY_ACT",
"DOMESTIC_VENDOR_NAME",
"DAVIS_BACON_ACT",
"VENDOR_RENEWAL_DATE",
"PARENT_MACOM_CODE",
"RESEARCH",
"UNDEFINITIZED_ACTION_DESC",
"DEPARTMENT_ID",
"TYPE_OF_SET_ASIDE_DESCRIPTION",
"AD_OTHER",
"FEE_PAID_FOR_USE_OF_SERVICE",
"PROPERTY_FINANCIAL_REPORTING",
"VENDOR_DUNS_NUMBER",
"WHO_CAN_USE_DESC",
"SOLICITATION_PROC_DESCRIPTION",
"AD_QUALITY_INSURANCE",
"FUNDING_OFFICE_NAME",
"CONTRACTING_AGENCY",
"SERVICE_CONTRACT_ACT_DESC",
"AWARD_STATUS",
"REASON_NOT_SB_DESCRIPTION",
"AD_CLOSEOUT",
"OBLIGATED_AMOUNT",
"REASON_NOT_COMP_DESCRIPTION",
"AGENCY_FULL_NAME",
"MULTIYEAR_CONTRACT",
"SYNOPSIS_WAIVER_EXCEPTION",
"DESCRIPTION_OF_REQUIREMENT",
"LAST_MODIFIED_BY",
"SUB_ACCOUNT_CODE",
"COMMERCIAL_ITEM_ACQUISTN_DESC",
"CONTRACT_BUNDLING",
"INHERENTLY_GOVERNMENTAL_DESC",
"NON_GOVERNMENT_VALUE",
"VENDOR_SITE_CODE",
"AD_ENGG_OR_PROD_SURVEILLANCE",
"PART8_OR_PART13",
"SBC_DEMONSTRATION_PROGRAM",
"ALTERNATIVE_ADVERTISING_DESC",
"CONSOLIDATED_CONTRACT_DESC",
"EVALUATED_PREF_DESCRIPTION",
"LOCATION_CODE",
"REF_IDV_CLASS_ID",
"PRINCIPAL_INV_LAST_NAME",
"ALT_PRINCIPAL_INV_MI_INITIAL",
"NAICS_DESCRIPTION",
"REF_IDV_AGENCY_ID",
"AD_TRANSPORTATION",
"MFG_ORGANIZATION_TYPE",
"SUBCONTRACT_PLAN",
"ORDERING_PROCEDURE",
"TYPE_OF_CONTRACT_PRICING",
"COST_ACCT_STANDARDS",
"ACCTNG_INSTALLATION_NUMBER",
"ADDITIONAL_REPORTING",
"MAIN_ACCOUNT_CODE",
"URL_OF_PROGRAM",
"ADMINISTRATOR_CODE",
"LOCAL_AREA_SET_ASIDE",
"AGENCY_NAME",
"CAS_CLAUSE_DESC",
"PRICE_EVAL_PERCENT_DIFF",
"ACCESSION_NUMBER",
"EXTENT_COMPETED_DESCRIPTION",
"FEE_RANGE_LOWER_VALUE",
"VENDOR_NAME",
"MULTIPLE_OR_SINGLE_DESC",
"TYPE_OF_FEE_FOR_USE_OF_SERVICE",
"AWARD_TYPE_DESCRIPTION",
"MAJOR_PROGRAM_CODE",
"PBSC_SHORT_DESCRIPTION",
"LASA_DESCRIPTION",
"CCR_EXCEPTION_TYPE_DESC",
"SOLICITATION_DATE",
"AD_BLANKET_DELEGATION",
"SYSTEM_EQUIPMENT_DESCRIPTION",
"INSTALLAION_UNIQUE",
"COMMERCIAL_ITEM_TEST_PROGRAM",
"COUNTRY_OF_INCORPORATION",
"CONTINGENCY_OPS_DESC",
"FINAL_INVOICE_PAID_DATE",
"MFG_ORGANIZATION_TYPE_DESC",
"TOTAL_DOLLARS_OBLIGATED",
"NASA_OTHER_STATUTORY_AUTHORITY",
"DOMESTIC_DUNS_NUMBER",
"REASON_FOR_MODIFICATION_DESC",
"FEE_RANGE_UPPER_VALUE",
"VENDOR_DOMESTIC_DUNS_NUMBER",
"FUNDED_BY_FOREIGN_ENTITY",
"CLINGER_COHEN_ACT_DESC",
"NUMBER_OF_OFFERS_RECEIVED",
"SEA_TRANSPORTATION_DESC",
"SOCIO_ECONOMIC_INDICATORS",
"ANNUAL_REVENUE",
"CONTRACT_TYPE",
"MGMT_REPORTING_REQUIREMENTS",
"AD_CONSENT_TO_SUBCONTRACT",
"NEW_TECH_PATENT_RIGHTS_CLAUSE",
"VENDOR_ADDRESS_STATE_NAME",
"PRODUCT_OR_SERVICE_DESCRIPTION",
"AWARD_COUNTRY_NAME",
"GLOBAL_VENDOR_NAME",
"LOCATION_NAME",
"LOCATION_STATE_NAME",
"SYS_LAST_MODIFIED_DATE",
"AWARD_TYPE",
"COST_OR_PRICING_DATA_DESC",
"COUNTY_NAME",
"CREATED_VIA",
"NUMBER_OF_ACTIONS",
"POM_SHORT_DESCRIPTION",
"POP_CONGRESS_DISTRICT_CODE",
"EXTENT_COMPETED",
"REF_IDV_PIID",
"REASON_NOT_SB",
"FED_BIZ_OPPS",
"PRODUCT_OR_SERVICE_TYPE",
"FUNDING_AGENCY_NAME",
"SOLICITATION_ID",
"ULTIMATE_CONTRACT_VALUE",
"PRIMARY_KEY",
"PRINCIPAL_NAICS_CODE",
"CONTRACT_FISCAL_YEAR",
"NATIONAL_INTEREST_CODE",
"TOTAL_ULTIMATE_CONTRACT_VALUE",
"TYPE_OF_FEE_DESC",
"ALT_PRINCIPAL_INV_FIRST_NAME",
"REGION_CODE",
"REASON_FOR_MODIFICATION",
"SIGNED_DATE",
"COUNTRY_OF_INCORP_NAME",
"DOLLARS_OBLIGATED",
"CLOSEOUT_PR",
"PIID",
"IT_COMMERCIAL_ITEM_CAT_DESC",
"SPENDING_CATEGORY",
"REF_IDV_AGENCY_NAME",
"PR_NUMBER",
"COTR_NAME",
"TYPE_OF_IDC_DESC",
"GLOBAL_DUNS_NUMBER",
"WHO_CAN_USE",
"FAIR_OPPORTUNITY_DESCRIPTION",
"FUNDED_THROUGH_DATE",
"FUNDING_DEPARTMENT_ID",
"SBC_DEMONSTRATION_PROGRAM_DESC",
"VALUE_ENGINEERING_CLAUSE",
"IT_COMMERCIAL_ITEM_CATEGORY",
"BASE_EXERCISED_OPTIONS_VALUE",
"TYPE_OF_SET_ASIDE",
"SOLICITATION_PROCEDURES",
"STATE_OF_INCORP_NAME",
"FUNDING_AGENCY_ID",
"PARENT_DUNS_NUMBER",
"VENDOR_ADDRESS_STATE_CODE",
"IS_PHYSICALLY_COMPLETE",
"CONTRACT_FUND_CODE",
"LOCATION_STATE_CODE",
"AGENCY_CODE",
"DEPARTMENT_FULL_NAME",
"AWARD_CLASS_ID",
"VENDOR_PHONE_NUMBER",
"ESTIMATED_COMPLETION_DATE",
"LAST_MODIFIED_DATE",
"TYPE_OF_IDC",
"VENDOR_PARENT_DUNS_NUMBER",
"TYPE_OF_AGREEMENT",
"ALTERNATE_COTR_NAME",
"IS_TRANSFERED",
"AD_COST_ACCOUNTING_STANDARDS",
"PROCUREMENT_PLACEMENT_CODE",
"FISCAL_YEAR",
"COUNTRY_CODE",
"CONTRACT_BUNDLING_DESC",
"EFFECTIVE_DATE",
"CANCELLATION_DATE",
"DAVIS_BACON_ACT_DESC",
"FUNDING_OFFICE_ID",
"LAST_DATE_TO_ORDER",
"VENDOR_ADDRESS_COUNTRY_CODE",
"PLACE_OF_PERFORMANCE_ZIP_CODE",
"PLACE_OF_PERFORM_COUNTRY_CODE",
"EVALUATED_PREFERENCE",
"PHYSICAL_COMPLETION_DATE",
"ADVISORY_ASSISTANCE_SERVICES",
"OTHER_STATUTORY_AUTHORITY",
"VENDOR_ADDRESS_ZIP_CODE",
"VENDOR_ADDRESS_COUNTRY_NAME",
"VENDOR_ULTIMATE_DUNS_NUMBER",
"VENDOR_ADDRESS_LINE_2",
"VENDOR_ADDRESS_LINE_1",
"VENDOR_ADDRESS_LINE_3",
"COUNTY_CODE",
"POP_STATE_NAME",
"ORGANIZATION_CODE",
"POP_COUNTRY_NAME",
"CONTINGENCY_OPS",
"SOLICITATION_ISSUE_DATE",
"CLAIMANT_PROGRAM",
"CONTRACTING_AUTHORITY_DESC",
"TOTAL_NON_GOVERNMENT_VALUE",
"MULTIPLE_OR_SINGLE_AWARD_IDC",
"VERSION",
"VENDOR_CONGRESS_DISTRICT_CODE",
"CLAIMANT_PROGRAM_DESCRIPTION",
"FED_BIZ_OPPS_DESC",
"CONTRACT_SUB_TYPE",
"A76_ACTION",
"STATE_OF_INCORPORATION",
"CONTRACTING_AGENCY_NAME",
"NEGOTIATOR_CODE",
"CONTRACT_FINANCING",
"RECOVERED_MATERIAL_CLAUSES",
"VENDOR_DIVISION_NUMBER",
"CREATED_DATE",
"PCARD_AS_PAYMENT_DESC",
"REASON_NOT_SDB_DESCRIPTION",
"VENDOR_ADDRESS_CITY",
"AGENCY_IDENTIFIER",
"TREASURY_ACCOUNT_SYMBOL",
"ALT_PRINCIPAL_INV_LAST_NAME",
"CONTRACTING_AUTHORITY",
"PRINCIPAL_INV_MIDDLE_INITIAL",
"DESTROY_DATE",
"CONSOLIDATED_CONTRACT",
"MULTIYEAR_CONTRACT_DESC",
"PURCHASE_REASON",
"VENDOR_DOING_BUSINESS_AS_NAME",
"SUBCONTRACT_PLAN_DESC",
"CONTRACTING_OFFICER_CODE",
"AWARD_COMPLETION_DATE",
"LETTER_CONTRACT",
"FUNDING_DEPARTMENT_NAME",
"FIXED_FEE_VALUE",
"TOTAL_CURRENT_CONTRACT_VALUE",
"PRE_AWARD_SYNOPSIS_DESC",
"GFE_GFP",
"ALTERNATIVE_ADVERTISING",
"CREATED_BY",
"CLINGER_COHEN_ACT",
"SYSTEM_EQUIPMENT_CODE",
"LAST_MOD_DATE",
"VENDOR_REGISTRATION_DATE",
"CONTRACT_PRICING_DESCRIPTION",
"ORIGINAL_AGENCY_CODE",
"FOREIGN_FUNDING_DESC",
"A76_DESC",
"VENDOR_FAX_NUMBER",
"COMMERCIAL_ITEM_ACQUISITION",
"VENDOR_FULL_NAME",
"PARENT_VENDOR_NAME",
"SUPPORT_SERVICES_TYPE_CONTRACT",
"ADDITIONAL_REPORTING_DESC",
"REASON_FOR_PURCHASE_DESC",
"RESEARCH_DESCRIPTION",
"OFFERORS_PROPOSAL_NUMBER",
"PCARD_AS_PAYMENT",
"VENDOR_DIVISION_NAME",
"DEPARTMENT_NAME",
"REF_IDV_MODIFICATION_NUMBER",
"CITP_DESC",
"FIELD_OF_SCIENCE_OR_ENGG",
"PLACE_OF_MANUFACTURE",
"ORGANIZATIONAL_TYPE",
"CCR_EXCEPTION_TYPE",
"WALSH_HEALEY_ACT_DESC",
"REVEAL_DATE",
"DOCUMENT_TYPE",
"GFE_GFP_DESC",
"CONTRACTING_OFFICE_NAME",
"USE_OF_EPA_PRODUCTS_DESC",
"USE_OF_EPA_PRODUCTS",
"PRE_AWARD_SYNOPSIS_REQUIREMENT",
"INHERENTLY_GOVERNMENTAL_CODE",
"MAXIMUM_ORDER_LIMIT",
"NATIONAL_INTEREST_DESCRIPTION",
"CO_BUS_SIZE_DETERMINATION_DESC",
"CO_BUS_SIZE_DETERMINATION",
"AD_SECURITY",
"CONTRACTING_AGENCY_ID",
"VENDOR_NAME_ALTERNATE",
"SEA_TRANSPORTATION",
"SWE_SHORT_DESCRIPTION",
"CONTRACTING_OFFICE_ID",
"STATRY_EXPTN_TO_FAIR_OPRTNTY",
"PERFORM_BASED_SERVICE_CONTRACT",
"AD_PROPERTY_ADMINISTRATION",
"COST_OR_PRICING_DATA",
"REASON_NOT_SDB",
"VENDOR_LEGAL_ORGANIZATION_NAME",
"NUMBER_OF_EMPLOYEES",
"MODIFICATION_NUMBER",
"NGC_PARTICIPATION"
)
parameter_name <-
c(
"contract_status_description",
"vendor_site_code_alternate",
"program_acronym",
"ad_post_award_audit",
"current_contract_value",
"award_type",
"ad_none",
"contractor_name",
"contract_financing_desc",
"security_code",
"transaction_number",
"nontraditional_government_contractor_participation",
"reason_not_competed",
"recovered_material_cls_desc",
"original_piid",
"principal_inv_first_name",
"email_address",
"psc",
"legislative_mandates",
"service_contract_act",
"walsh_healey_act",
"domestic_vendor_name",
"davis_bacon_act",
"vendor_renewal_date",
"parent_macom_code",
"research",
"undefinitized_action_desc",
"department_id",
"type_of_set_aside",
"ad_other",
"fee_paid_for_use_of_service",
"property_financial_reporting",
"vendor_duns",
"who_can_use_desc",
"solicitation_procedures",
"ad_quality_insurance",
"funding_office_name",
"contracting_agency",
"service_contract_act_desc",
"award_status",
"reason_not_sb_description",
"ad_closeout",
"action_obligation",
"other_than_full_and_open_competition",
"agency_full_name",
"multiyear_contract",
"synopsis_waiver_exception",
"description_of_requirement",
"last_modified_by",
"treasury_account_symbol_sub_account_code",
"commercial_item_acquisition_procedures",
"contract_bundling",
"inherently_governmental_description",
"non-government_dollars",
"vendor_site_code",
"ad_engg_or_prod_surveillance",
"part8_or_part13",
"small_business_competitiveness_demonstration_program",
"alternative_advertising_desc",
"consolidated_contract_desc",
"evaluated_preference",
"location_code",
"ref_idv_class_id",
"principal_inv_last_name",
"alt_principal_inv_mi_initial",
"naics_description",
"reference_idv_agency_id",
"ad_transportation",
"mfg_organization_type",
"subcontract_plan",
"ordering_procedure",
"type_of_contract_pricing",
"cost_acct_standards",
"acctng_installation_number",
"additional_reporting_code",
"treasury_account_symbol_main_account_code",
"url_of_program",
"administrator_code",
"local_area_set_aside",
"agency_name",
"cas_clause_desc",
"price_eval_percent_diff",
"accession_number",
"extent_competed",
"fee_range_lower_value",
"vendor_name",
"multiple_or_single_desc",
"type_of_fee_for_use_of_service",
"award_type",
"major_program_code",
"pbsc_short_description",
"lasa_description",
"ccr_exception_type_desc",
"solicitation_date",
"ad_blanket_delegation",
"system_equipment_description",
"installaion_unique",
"commercial_item_test_program",
"country_of_incorporation",
"contingency_ops_desc",
"final_invoice_paid_date",
"mfg_organization_type_desc",
"total_dollars_obligated",
"nasa_other_statutory_authority",
"domestic_duns_number",
"reason_for_modification_desc",
"fee_range_upper_value",
"vendor_domestic_duns_number",
"funded_by_foreign_entity",
"clinger_cohen_act_desc",
"number_of_offers_received",
"sea_transportation_desc",
"socio_economic_indicators",
"annual_revenue",
"contract_type",
"mgmt_reporting_requirements",
"ad_consent_to_subcontract",
"new_tech_patent_rights_clause",
"vendor_state_name",
"psc_description",
"award_country_name",
"global_vendor_name",
"location_name",
"location_state_name",
"sys_last_modified_date",
"award_type",
"cost_or_pricing_data_desc",
"county_name",
"created_via",
"number_of_actions",
"pom_short_description",
"pop_congressional_district_code",
"extent_competed",
"reference_idv",
"reason_not_sb",
"fed_biz_opps",
"psc_type",
"funding_agency_name",
"solicitation_id",
"ultimate_contract_value",
"primary_key",
"naics",
"contract_fiscal_year",
"national_interest_code",
"total_base_and_exercised_options_value",
"type_of_fee_desc",
"alt_principal_inv_first_name",
"region_code",
"reason_for_modification",
"date_signed",
"country_of_incorp_name",
"dollars_obligated",
"closeout_pr",
"contract_id",
"it_commercial_item_cat_desc",
"initiative",
"ref_idv_agency_name",
"pr_number",
"cotr_name",
"type_of_idc_desc",
"global_duns_number",
"who_can_use",
"fair_opportunity",
"funded_through_date",
"funding_department_id",
"small_business_competitiveness_demonstration_program",
"value_engineering_clause",
"it_commercial_item_category",
"base_and_exercised_options_value",
"type_of_set_aside",
"solicitation_procedures",
"state_of_incorp_name",
"funding_agency_id",
"parent_duns_number",
"vendor_state",
"is_physically_complete",
"contract_fund_code",
"location_state_code",
"agency_code",
"department_full_name",
"award_class_id",
"vendor_phone_number",
"estimated_completion_date",
"last_modified_date",
"type_of_idc",
"vendor_parent_duns_number",
"type_of_agreement",
"alternate_cotr_name",
"is_transfered",
"ad_cost_accounting_standards",
"procurement_placement_code",
"other_transaction_fiscal_year",
"country_code",
"contract_bundling_desc",
"effective_date",
"cancellation_date",
"davis_bacon_act_desc",
"funding_office_id",
"last_date_to_order",
"vendor_country_code",
"place_of_performance_zip_code",
"place_of_perform_country_code",
"evaluated_preference",
"physical_completion_date",
"advisory_assistance_services",
"other_statutory_authority",
"vendor_zip_code",
"vendor_country_name",
"vendor_ultimate_duns_number",
"vendor_address_line_2",
"vendor_address_line_1",
"vendor_address_line_3",
"county_code",
"pop_state_name",
"organization_code",
"pop_country_name",
"contingency_ops",
"solicitation_issue_date",
"claimant_program",
"contracting_authority_desc",
"total_non_government_value",
"multiple_or_single_award_idc",
"version",
"vendor_congressional_district_code",
"claimant_program_description",
"fed_biz_opps",
"award_type",
"a76_action",
"state_of_incorporation",
"contracting_agency_name",
"negotiator_code",
"contract_financing",
"recovered_material_clauses",
"vendor_division_number",
"created_date",
"pcard_as_payment_desc",
"reason_not_sdb_description",
"vendor_city",
"treasury_account_symbol_agency_identifier",
"treasury_account_symbol",
"alt_principal_inv_last_name",
"contracting_authority",
"principal_inv_middle_initial",
"destroy_date",
"consolidated_contract",
"multiyear_contract_desc",
"purchase_reason",
"vendor_doing_business_as_name",
"subcontract_plan_desc",
"contracting_officer_code",
"award_completion_date",
"letter_contract",
"funding_department_name",
"fixed_fee_value",
"total_base_and_all_options_value",
"pre_award_synopsis_desc",
"gfe_gfp",
"alternative_advertising",
"created_by",
"clinger_cohen_act",
"system_equipment_code",
"last_modified_date",
"vendor_registration_date",
"contract_pricing_description",
"original_agency_code",
"foreign_funding_desc",
"a76",
"vendor_fax_number",
"commercial_item_acquisition",
"vendor_full_name",
"parent_vendor_name",
"support_services_type_contract",
"additional_reporting_description",
"reason_for_purchase_desc",
"sbir_sttr",
"offerors_proposal_number",
"pcard_as_payment",
"vendor_division_name",
"department_name",
"reference_idv_modification_number",
"commercial_item_test_program",
"field_of_science_or_engg",
"place_of_manufacture",
"organizational_type",
"ccr_exception_type",
"walsh_healey_act_desc",
"reveal_date",
"document_type",
"gfe_gfp_desc",
"contracting_office_name",
"use_of_epa_products_desc",
"use_of_epa_products",
"pre_award_synopsis_requirement",
"inherently_governmental_code",
"maximum_order_limit",
"national_interest_description",
"co_bus_size_determination_desc",
"co_bus_size_determination",
"ad_security",
"contracting_agency_id",
"vendor_name_alternate",
"sea_transportation",
"swe_short_description",
"contracting_office_id",
"statry_exptn_to_fair_oprtnty",
"perform_based_service_contract",
"ad_property_administration",
"cost_or_pricing_data",
"reason_not_sdb",
"vendor_legal_organization_name",
"number_of_employees",
"modification_number",
"ngc_participation"
)
tibble(parameter, parameter_name)
}
# https://www.fpds.gov/downloads/Version_1.4_specs/FPDSNG_DataDictionary_V1.4.pdf
#' FPDS name dictionary
#'
#' @return
#' @export
#'
#' @examples
dictionary_fpds_names <-
memoise::memoise(function() {
nameFPDS = c(
"ref_idv_contract_id",
"parent_vendor_name",
"subcontract_plan",
"solicitation_proc_description",
"award_type",
"awardIDagencyID",
"awardIDagencyIDDescription",
"awardIDPIID",
"awardIDmodNumber",
"awardIDtransactionNumber",
"relevantContractDatessignedDate",
"relevantContractDateseffectiveDate",
"relevantContractDatescurrentCompletionDate",
"relevantContractDatesultimateCompletionDate",
"dollarValuesobligatedAmount",
"dollarValuesbaseAndExercisedOptionsValue",
"dollarValuesbaseAndAllOptionsValue",
"totalDollarValuestotalObligatedAmount",
"totalDollarValuestotalBaseAndExercisedOptionsValue",
"totalDollarValuestotalBaseAndAllOptionsValue",
"purchaserInformationcontractingOfficeAgencyID",
"purchaserInformationcontractingOfficeAgencyIDDescription",
"purchaserInformationcontractingOfficeID",
"purchaserInformationcontractingOfficeIDDescription",
"purchaserInformationfundingRequestingAgencyID",
"purchaserInformationfundingRequestingAgencyIDDescription",
"purchaserInformationfundingRequestingOfficeID",
"purchaserInformationfundingRequestingOfficeIDDescription",
"purchaserInformationforeignFunding",
"purchaserInformationforeignFundingDescription",
"contractMarketingDatafeePaidForUseOfService",
"contractDatacontractActionType",
"contractDatacontractActionTypeDescription",
"contractDatatypeOfContractPricing",
"contractDatatypeOfContractPricingDescription",
"contractDatanationalInterestActionCode",
"contractDatanationalInterestActionCodeDescription",
"contractDatadescriptionOfContractRequirement",
"contractDatainherentlyGovernmentalFunction",
"contractDatainherentlyGovernmentalFunctionDescription",
"contractDataGFEGFP",
"contractDataGFEGFPDescription",
"contractDataseaTransportation",
"contractDataseaTransportationDescription",
"contractDataundefinitizedAction",
"contractDataundefinitizedActionDescription",
"contractDataconsolidatedContract",
"contractDataconsolidatedContractDescription",
"contractDataperformanceBasedServiceContract",
"contractDataperformanceBasedServiceContractDescription",
"contractDatacontingencyHumanitarianPeacekeepingOperation",
"contractDatacontingencyHumanitarianPeacekeepingOperationDescription",
"contractDatalistOfTreasuryAccounts",
"contractDatacontractFinancing",
"contractDatacontractFinancingDescription",
"contractDatapurchaseCardAsPaymentMethod",
"contractDatapurchaseCardAsPaymentMethodDescription",
"contractDatanumberOfActions",
"legislativeMandatesClingerCohenAct",
"legislativeMandatesClingerCohenActDescription",
"legislativeMandatesmaterialsSuppliesArticlesEquipment",
"legislativeMandatesmaterialsSuppliesArticlesEquipmentDescription",
"legislativeMandateslaborStandards",
"legislativeMandateslaborStandardsDescription",
"legislativeMandatesconstructionWageRateRequirements",
"legislativeMandatesconstructionWageRateRequirementsDescription",
"legislativeMandatesinteragencyContractingAuthority",
"productOrServiceInformationproductOrServiceCode",
"legislativeMandatesinteragencyContractingAuthorityDescription",
"productOrServiceInformationcontractBundling",
"productOrServiceInformationproductOrServiceCodeDescription",
"productOrServiceInformationclaimantProgramCode",
"productOrServiceInformationcontractBundlingDescription",
"productOrServiceInformationprincipalNAICSCode",
"productOrServiceInformationclaimantProgramCodeDescription",
"productOrServiceInformationrecoveredMaterialClauses",
"productOrServiceInformationprincipalNAICSCodeDescription",
"productOrServiceInformationmanufacturingOrganizationType",
"productOrServiceInformationrecoveredMaterialClausesDescription",
"productOrServiceInformationsystemEquipmentCode",
"productOrServiceInformationmanufacturingOrganizationTypeDescription",
"productOrServiceInformationinformationTechnologyCommercialItemCategory",
"productOrServiceInformationsystemEquipmentCodeDescription",
"productOrServiceInformationuseOfEPADesignatedProducts",
"productOrServiceInformationinformationTechnologyCommercialItemCategoryDescription",
"productOrServiceInformationcountryOfOrigin",
"productOrServiceInformationuseOfEPADesignatedProductsDescription",
"productOrServiceInformationplaceOfManufacture",
"productOrServiceInformationcountryOfOriginDescription",
"vendorisAlaskanNativeOwnedBusinessCorporationOrFirm",
"productOrServiceInformationplaceOfManufactureDescription",
"vendorisAmericanIndianOwnedBusinessBusinessBusiness",
"vendorisIndianTribe",
"vendorisNativeHawaiianOwnedBusinessOrganizationOrFirm",
"vendorisTriballyOwnedBusinessBusinessFirm",
"vendorisSmallBusiness",
"vendorisVeteranOwnedBusiness",
"vendorisServiceDisabledVeteranOwnedBusinessBusiness",
"vendorisWomanOwned",
"vendorminorityOwned",
"vendorisVerySmallBusiness",
"vendorisWomanOwnedSmallBusiness",
"vendorisEconomicallyDisadvantagedWomanOwnedSmallBusiness",
"vendorisJointVentureWomenOwnedSmallBusiness",
"vendorisJointVentureWomenOwnedSmallBusiness",
"vendorisCommunityDevelopedCorporationOwnedFirm",
"vendorisLaborSurplusAreaFirm",
"vendorfederalGovernment",
"vendorisStateGovernment",
"vendorlocalGovernment",
"vendorisTribalGovernment",
"vendorisForeignGovernment",
"vendorbusinessOrOrganizationType",
"vendorisArchitectureAndEngineering",
"vendorisCommunityDevelopmentCorporation",
"vendorisConstructionFirm",
"vendorisDomesticShelter",
"vendorisEducationalInstitution",
"vendorisFoundation",
"vendorisHospital",
"vendorisManufacturerOfGoods",
"vendorisResearchAndDevelopment",
"vendorisServiceProvider",
"vendorisVeterinaryHospital",
"vendorisHispanicServicingInstitution",
"vendorreceivesContracts",
"vendorreceivesGrants",
"vendorreceivesContractsAndGrants",
"vendorisAirportAuthority",
"vendorisCouncilOfGovernments",
"vendorisHousingAuthoritiesPublicOrTribal",
"vendorisInterstateEntity",
"vendorisPlanningCommission",
"vendorisPortAuthority",
"vendorisTransitAuthority",
"vendorisSubchapterSCorporation",
"vendorisLimitedLiabilityCorporation",
"vendorisForeignOwnedAndLocated",
"vendorprofitStructure",
"vendorisShelteredWorkshop",
"vendorstateOfIncorporation",
"vendorcountryOfIncorporation",
"vendorstateOfIncorporationDescription",
"vendororganizationalType",
"vendorcountryOfIncorporationDescription",
"vendoris1862LandGrantCollege",
"vendoris1890LandGrantCollege",
"vendoris1994LandGrantCollege",
"vendorisHistoricallyBlackCollegeOrUniversity",
"vendorisMinorityInstitution",
"vendorisPrivateUniversityOrCollege",
"vendorisSchoolOfForestry",
"vendorisStateControlledInstitutionOfHigherLearning",
"vendorisTribalCollege",
"vendorisVeterinaryCollege",
"vendorisAlaskanNativeServicingInstitution",
"vendorisNativeHawaiianServicingInstitution",
"vendorisDOTCertifiedDisadvantagedBusinessEnterprise",
"vendorisSelfCertifiedSmallDisadvantagedBusiness",
"vendorisSBACertifiedSmallDisadvantagedBusiness",
"vendorisSBACertified8AProgramParticipant",
"vendorisSelfCertifiedHUBZoneJointVenture",
"vendorisSBACertifiedHUBZone",
"vendorisSBACertified8AJointVenture",
"vendorstreetAddress",
"vendorcity",
"vendorstate",
"vendorZIPCode",
"vendorstateDescription",
"vendorcountryCode",
"vendorZIPCodeDescription",
"vendorphoneNo",
"vendorcountryCodeDescription",
"vendorfaxNo",
"vendorcongressionalDistrictCode",
"vendortext",
"vendorDUNSNumber",
"vendorvendorName",
"vendorglobalParentDUNSNumber",
"vendorglobalParentDUNSName",
"vendorregistrationDate",
"vendorrenewalDate",
"placeOfPerformancestateCode",
"placeOfPerformancecountryCode",
"placeOfPerformanceplaceOfPerformanceZIPCode",
"placeOfPerformanceplaceOfPerformanceCongressionalDistrict",
"competitionextentCompeted",
"competitionsolicitationProcedures",
"competitionextentCompetedDescription",
"competitiontypeOfSetAside",
"competitionsolicitationProceduresDescription",
"competitionevaluatedPreference",
"competitiontypeOfSetAsideDescription",
"competitionresearch",
"competitionevaluatedPreferenceDescription",
"competitionnumberOfOffersReceived",
"competitionresearchDescription",
"competitioncommercialItemAcquisitionProcedures",
"competitioncommercialItemTestProgram",
"competitioncommercialItemAcquisitionProceduresDescription",
"competitionA76Action",
"competitioncommercialItemTestProgramDescription",
"competitionfedBizOpps",
"competitionA76ActionDescription",
"competitionlocalAreaSetAside",
"competitionfedBizOppsDescription",
"preferenceProgramssubcontractPlan",
"competitionlocalAreaSetAsideDescription",
"transactionInformationcreatedBy",
"preferenceProgramssubcontractPlanDescription",
"transactionInformationcreatedDate",
"transactionInformationlastModifiedBy",
"transactionInformationlastModifiedDate",
"transactionInformationstatus",
"transactionInformationapprovedBy",
"transactionInformationstatusDescription",
"transactionInformationapprovedDate",
"vendorisMinorityOwnedBusiness",
"vendorisSubcontinentAsianIndianAmericanOwnedBusiness",
"vendorisAsianPacificAmericanOwnedBusiness",
"vendorisBlackAmericanOwnedBusiness",
"vendorisHispanicAmericanOwnedBusiness",
"vendorisNativeAmericanOwnedBusiness",
"vendorisOtherMinorityOwned",
"vendorisFederalGovernment",
"vendorisFederallyFundedResearchAndDevelopmentCorp",
"vendorisFederalGovernmentAgency",
"vendorisLocalGovernment",
"vendorisCityLocalGovernment",
"vendorisCountyLocalGovernment",
"vendorisInterMunicipalLocalGovernment",
"vendorisLocalGovernmentOwned",
"vendorisMunicipalityLocalGovernment",
"vendorisSchoolDistrictLocalGovernment",
"vendorisTownshipLocalGovernment",
"vendorisCorporateEntityNotTaxExempt",
"vendorisCorporateEntityTaxExempt",
"vendorisPartnershipOrLimitedLiabilityPartnership",
"vendorisSolePropreitorship",
"vendorisSmallAgriculturalCooperative",
"vendorisInternationalOrganization",
"vendorisOtherbusinessOrOrganization",
"vendorisUSGovernmentEntity",
"vendorisForProfitOrganization",
"vendorisNonprofitOrganization",
"vendorisOtherNotForProfitOrganization",
"nameContract",
"urlFPDSContractAtom",
"datetimeContractModified",
"contractDatareasonForModification",
"contractDatareasonForModificationDescription",
"contractDatacostOrPricingData",
"contractDatacostOrPricingDataDescription",
"contractDatacostAccountingStandardsClause",
"contractDatacostAccountingStandardsClauseDescription",
"contractDatamultiYearContract",
"contractDatamultiYearContractDescription",
"placeOfPerformancelocationCode",
"competitionsmallBusinessCompetitivenessDemonstrationProgram",
"competitionpriceEvaluationPercentDifference",
"contractDatasolicitationID",
"legislativeMandateslistOfAdditionalReportingValues",
"vendorcageCode",
"competitiontypeOfSetAsideSource",
"competitiontypeOfSetAsideSourceDescription",
"competitionnumberOfOffersSource",
"competitionnumberOfOffersSourceDescription",
"transactionInformationclosedStatus",
"contractIDagencyID",
"contractIDagencyIDDescription",
"contractIDPIID",
"contractIDmodNumber",
"relevantContractDateslastDateToOrder",
"dollarValuestotalEstimatedOrderValue",
"contractMarketingDatawebsiteURL",
"contractMarketingDatawhoCanUse",
"contractMarketingDataemailAddress",
"contractMarketingDataindividualOrderLimit",
"contractDatamultipleOrSingleAwardIDC",
"contractDatamultipleOrSingleAwardIDCDescription",
"legislativeMandatesotherStatutoryAuthority",
"contractDatareferencedIDVMultipleOrSingle",
"contractDatareferencedIDVMultipleOrSingleDescription",
"contractDatareferencedIDVType",
"contractDatareferencedIDVTypeDescription",
"contractMarketingDatatypeOfFeeForUseOfService",
"contractMarketingDatatypeOfFeeForUseOfServiceDescription",
"contractDatamajorProgramCode",
"contractDatatypeOfIDC",
"contractDatatypeOfIDCDescription",
"contractDataprogramAcronym",
"contractMarketingDatafeeRangeLowerValue",
"contractMarketingDatafeeRangeUpperValue",
"contractMarketingDataorderingProcedure",
"competitionstatutoryExceptionToFairOpportunity",
"competitionstatutoryExceptionToFairOpportunityDescription",
"transactionInformationclosedBy",
"transactionInformationclosedDate",
"competitionreasonNotCompeted",
"competitionreasonNotCompetedDescription",
"contractDatareferencedIDVPart8OrPart13",
"vendorstreetAddress2",
"competitionidvTypeOfSetAside",
"competitionidvTypeOfSetAsideDescription",
"competitionidvNumberOfOffersReceived",
"genericTagsgenericStrings",
"agencySpecificAwardElementstext",
"agencySpecificAwardElementsPRNumber",
"agencySpecificAwardElementscloseoutPR",
"agencySpecificAwardElementsprocurementPlacementCode",
"agencySpecificAwardElementsadvisoryOrAssistanceServicesContract",
"agencySpecificAwardElementssupportServicesTypeContract",
"agencySpecificAwardElementsnewTechnologyOrPatentRightsClause",
"agencySpecificAwardElementsmanagementReportingRequirements",
"agencySpecificAwardElementspropertyFinancialReporting",
"agencySpecificAwardElementsvalueEngineeringClause",
"agencySpecificAwardElementssecurityCode",
"agencySpecificAwardElementsadministratorCode",
"agencySpecificAwardElementscontractingOfficerCode",
"agencySpecificAwardElementsnegotiatorCode",
"agencySpecificAwardElementsCOTRName",
"agencySpecificAwardElementsorganizationCode",
"agencySpecificAwardElementscontractFundCode",
"agencySpecificAwardElementsisPhysicallyComplete",
"agencySpecificAwardElementsaccountingInstallationNumber",
"agencySpecificAwardElementsfundedThroughDate",
"agencySpecificAwardElementsprincipalInvestigator",
"agencySpecificAwardElementsfieldOfScienceOrEngineering",
"agencySpecificAwardElementsphysicalCompletionDate",
"agencySpecificAwardElementsaccessionNumber",
"agencySpecificAwardElementsdestroyDate",
"agencySpecificAwardElementssolicitationIssueDate",
"agencySpecificAwardElementslistOfContractAdministrationsDelegated",
"agencySpecificAwardElementsfinalInvoicePaidDate",
"contractMarketingDatafixedFeeValue",
"legislativeMandatesWalshHealyAct",
"legislativeMandatesWalshHealyActDescription",
"legislativeMandatesserviceContractAct",
"legislativeMandatesserviceContractActDescription",
"legislativeMandatesDavisBaconAct",
"legislativeMandatesDavisBaconActDescription",
"piid",
"ref_idv_piid",
"ref_idv_agency_id",
"modification_number",
"contracting_agency_id",
"contracting_agency_name",
"contracting_office_id",
"contracting_office_name",
"funding_agency_id",
"funding_office_id",
"funding_office_name",
"agency_code",
"agency_name",
"department_id",
"department_name",
"last_mod_date",
"last_modified_by",
"award_completion_date",
"created_date",
"signed_date",
"effective_date",
"estimated_completion_date",
"obligated_amount",
"ultimate_contract_value",
"type_of_contract_pricing",
"award_status",
"contract_type",
"created_by",
"description_of_requirement",
"reason_for_modification",
"legislative_mandates",
"local_area_set_aside",
"socio_economic_indicators",
"multiyear_contract",
"national_interest_code",
"national_interest_description",
"principal_naics_code",
"naics_description",
"product_or_service_code",
"product_or_service_description",
"pop_congress_district_code",
"pop_congress_country",
"pop_state_name",
"vendor_address_city",
"vendor_congress_district_code",
"vendor_address_country_code",
"vendor_address_country_name",
"vendor_duns_number",
"vendor_doing_business_as_name",
"vendor_name",
"vendor_address_state_code",
"vendor_address_state_name",
"vendor_address_zip_code",
"global_vendor_name",
"vendortreasuryAccountSymbol",
"agencySpecificAwardElementsinstallationUnique",
"Contract ID",
"Reference IDV",
"Modification Number",
"Transaction Number",
"Award/IDV Type",
"Action Obligation ($)",
"Date Signed",
"Solicitation Date",
"Contracting Agency ID",
"Contracting Agency",
"Contracting Office Name",
"PSC Type",
"PSC",
"PSC Description",
"NAICS",
"NAICS Description",
"Vendor DUNS",
"Vendor Name",
"Vendor City",
"Vendor State",
"Vendor ZIP Code",
"Global DUNS Number",
"Global Vendor Name",
"Additional Reporting Code",
"Additional Reporting Description",
"CGAC AGENCY CODE",
"FPDS DEPARTMENT ID",
"AGENCY NAME",
"AGENCY ABBREVIATION",
"REGISTERED IN BROKER",
"REGISTERED IN ASP",
"ORIGINAL SOURCE FOR CGAC or SubTier",
"SUBTIER CODE",
"SUBTIER NAME",
"SUBTIER ABBREVIATION",
"Admin Org Name",
"ADMIN_ORG",
"FREC",
"FREC Entity Description",
"SubTierAcronymSource",
"SubTier appears in data in FPDS or ASP?",
"REGISTERED IN ASP_1",
"AGENCY NAME (ORIGINAL)",
"SUBTIER NAME (ORIGINAL)",
"IS_FREC",
"MISSION",
"WEBSITE",
"ICON FILENAME",
"Agency Code",
"Agency Name",
"Bureau Code",
"Bureau Name",
"Account Code",
"Account Name",
"Treasury Agency Code",
"Subfunction Code",
"Subfunction Title",
"BEA Category",
"On- or Off- Budget",
"1976",
"TQ",
"1977",
"1978",
"1979",
"1980",
"1981",
"1982",
"1983",
"1984",
"1985",
"1986",
"1987",
"1988",
"1989",
"1990",
"1991",
"1992",
"1993",
"1994",
"1995",
"1996",
"1997",
"1998",
"1999",
"2000",
"2001",
"2002",
"2003",
"2004",
"2005",
"2006",
"2007",
"2008",
"2009",
"2010",
"2011",
"2012",
"2013",
"2014",
"2015",
"2016",
"2017",
"2018",
"2019",
"2020",
"2021",
"2022",
"pop_country_name",
"parent_duns_number",
"research",
"Funding Agency Name",
"Funding Agency ID",
"Place of Manufacture Description",
"Country of Product or Service Origin Code",
"PIID",
"Completion Date",
"NAICS Code",
"Product or Service Code",
"Product or Service Description",
"Vendor Country",
"DUNS Number",
"Type of Contract",
"Award or IDV Type",
"Base and All Options Value (Total Contract Value)",
"Action Obligation",
"Title",
"Sol #",
"Agency",
"Office",
"Location",
"Type",
"Orig. Type",
"Last Posting Date",
"Orig. Posting Date",
"Response Deadline",
"Archiving Policy",
"Archive Date",
"Set-aside Type",
"Classification Code",
"Naics Code",
"Office Address",
"Contact Info",
"Place of Performance",
"Additional Info Link",
"Description",
"Active",
"Link",
"Award #",
"Posting Date",
"Award Date",
"Award Amount",
"Line Number",
"Awardee",
"numberItem",
"DATE",
"AGENCY",
"OFFICE",
"LOCATION",
"ZIP",
"CLASSCOD",
"OFFADD",
"SUBJECT",
"SOLNBR",
"RESPDATE",
"CONTACT",
"DESC",
"URL",
"POPCOUNTRY",
"POPZIP",
"POPADDRESS",
"detailsSolicitation",
"ARCHDATE",
"EMAIL",
"SETASIDE",
"NTYPE",
"AWDNBR",
"AWDAMT",
"AWDDATE",
"AWARDEE",
"LINENBR",
"BR",
"Company Name",
"STAUTH",
"MODNBR",
"FOJA",
"DONBR",
"ADDRESS",
"CORRECTION",
"LINK",
"RECOVERY_ACT",
"DOCUMENT_PACKAGES",
"CHANGES",
"vendorisWomanOwnedBusiness",
"vendorisWomanOwnedBusinessSmallBusiness",
"agencySpecificIDVElementsagencyID",
"agencySpecificIDVElementsNASASpecificIDVElements",
"CONGRESSIONAL JUSTIFICATION",
"awardContractID.agencyID",
"awardContractID.PIID",
"awardContractID.modNumber",
"awardContractID.transactionNumber",
"referencedIDVID.agencyID",
"referencedIDVID.PIID",
"referencedIDVID.modNumber",
"signedDate",
"effectiveDate",
"currentCompletionDate",
"ultimateCompletionDate",
"obligatedAmount",
"baseAndExercisedOptionsValue",
"baseAndAllOptionsValue",
"contractingOfficeAgencyID",
"contractingOfficeID",
"fundingRequestingAgencyID",
"fundingRequestingOfficeID",
"foreignFunding",
"feePaidForUseOfService",
"contractActionType",
"typeOfContractPricing",
"reasonForModification",
"majorProgramCode",
"nationalInterestActionCode",
"descriptionOfContractRequirement",
"inherentlyGovernmentalFunction",
"GFE-GFP",
"undefinitizedAction",
"consolidatedContract",
"performanceBasedServiceContract",
"contingencyHumanitarianPeacekeepingOperation",
"treasuryAccount.agencyIdentifier",
"treasuryAccount.mainAccountCode",
"treasuryAccount.obligatedAmount",
"purchaseCardAsPaymentMethod",
"numberOfActions",
"ClingerCohenAct",
"materialsSuppliesArticlesEquipment",
"laborStandards",
"constructionWageRateRequirements",
"interagencyContractingAuthority",
"productOrServiceCode",
"contractBundling",
"principalNAICSCode",
"recoveredMaterialClauses",
"manufacturingOrganizationType",
"useOfEPADesignatedProducts",
"countryOfOrigin",
"placeOfManufacture",
"vendorName",
"isAlaskanNativeOwnedBusinessCorporationOrFirm",
"isAmericanIndianOwnedBusinessBusinessBusiness",
"isIndianTribe",
"isNativeHawaiianOwnedBusinessOrganizationOrFirm",
"isTriballyOwnedBusinessBusinessFirm",
"isVeteranOwnedBusiness",
"isServiceDisabledVeteranOwnedBusinessBusiness",
"isWomanOwnedBusiness",
"isMinorityOwnedBusiness",
"isSubcontinentAsianIndianAmericanOwnedBusiness",
"isAsianPacificAmericanOwnedBusiness",
"isBlackAmericanOwnedBusiness",
"isHispanicAmericanOwnedBusiness",
"isNativeAmericanOwnedBusiness",
"isOtherMinorityOwnedBusiness",
"isVerySmallBusiness",
"isWomanOwnedBusinessSmallBusiness",
"isEconomicallyDisadvantagedWomanOwnedSmallBusiness",
"isJointVentureWomenOwnedSmallBusiness",
"isJointVentureWomenOwnedSmallBusiness",
"isCommunityDevelopedCorporationOwnedFirm",
"isLaborSurplusAreaFirm",
"isUSFederalGovernment",
"isFederallyFundedResearchAndDevelopmentCorp",
"isFederalAgency",
"isStateGovernment",
"isLocalGovernment",
"isCityGovernment",
"isCountyGovernment",
"isInterMunicipalLocalGovernment",
"isLocalGovernment",
"isSchoolDistrictLocalGovernment",
"isSchoolDistrictLocalGovernment",
"isTownshipLocalGovernment",
"isTribalGovernment",
"isForeignGovernment",
"isCorporateEntityNonTaxExempt",
"isCorporateEntityTaxExempt",
"isPartnershipOrLLP",
"isSolePropreitorship",
"isSmallAgriculturalCooperative",
"isInternationalOrganization",
"isCommunityDevelopedCorporationOwnedFirm",
"isDomesticShelter",
"isEducationalInstitution",
"isFoundation",
"isHospital",
"isManufacturerOfGoods",
"isVeterinaryHospital",
"isHispanicServicingInstitution",
"receivesContracts",
"receivesGrants",
"receivesContractsAndGrants",
"isAirportAuthority",
"isCouncilOfGovernments",
"isHousingAuthoritiesPublicOrTribal",
"isInterstateEntity",
"isPlanningCommission",
"isPortAuthority",
"isTransitAuthority",
"isSubchapterSCorporation",
"isLimitedLiabilityCorporation",
"isForeignOwnedAndLocated",
"isForProfitOrganization",
"isNonprofitOrganization",
"isOtherNotForProfitOrganization",
"isShelteredWorkshop",
"stateOfIncorporation",
"countryOfIncorporation",
"organizationalType",
"is1862LandGrantCollege",
"is1890LandGrantCollege",
"is1994LandGrantCollege",
"isHistoricallyBlackCollegeOrUniversity",
"isMinorityInstitution",
"isPrivateUniversityOrCollege",
"isSchoolOfForestry",
"isStateControlledInstitutionOfHigherLearning",
"isTribalCollege",
"isVeterinaryCollege",
"isDOTCertifiedDisadvantagedBusinessEnterprise",
"isSelfCertifiedSmallDisadvantagedBusiness",
"isSBACertifiedSmallDisadvantagedBusiness",
"isSBACertified8AProgramParticipant",
"isSelfCertifiedHUBZoneJointVenture",
"isSBACertifiedHUBZone",
"isSBACertified8AJointVenture",
"vendorLocation.streetAddress",
"vendorLocation.city",
"vendorLocation.state",
"vendorLocation.ZIPCode",
"vendorLocation.countryCode",
"vendorLocation.phoneNo",
"vendorLocation.faxNo",
"vendorLocation.congressionalDistrictCode",
"vendorSiteCode",
"vendorAlternateSiteCode",
"vendorDUNSInformation.DUNSNumber",
"vendorDUNSInformation.cageCode",
"ccrRegistrationDetails.registrationDate",
"ccrRegistrationDetails.renewalDate",
"contractingOfficerBusinessSizeDetermination",
"principalPlaceOfPerformance.stateCode",
"principalPlaceOfPerformance.countryCode",
"placeOfPerformanceZIPCode",
"placeOfPerformanceCongressionalDistrict",
"extentCompeted",
"solicitationProcedures",
"typeOfSetAside",
"typeOfSetAsideSource",
"evaluatedPreference",
"numberOfOffersReceived",
"numberOfOffersSource",
"commercialItemAcquisitionProcedures",
"A76Action",
"localAreaSetAside",
"createdBy",
"createdDate",
"lastModifiedBy",
"lastModifiedDate",
"closedBy",
"closedDate",
"closedStatus",
"status",
"listOfAdditionalReportingValues.additionalReportingValue",
"vendorLegalOrganizationName",
"idvTypeOfSetAside",
"statutoryExceptionToFairOpportunity",
"idvNumberOfOffersReceived",
"commercialItemTestProgram",
"fedBizOpps",
"vendorAlternateName",
"costOrPricingData",
"multiYearContract",
"reasonNotCompeted",
"subcontractPlan",
"solicitationID",
"contractFinancing",
"vendorDoingAsBusinessName",
"informationTechnologyCommercialItemCategory",
"costAccountingStandardsClause",
"priceEvaluationPercentDifference",
"otherStatutoryAuthority",
"IDVID.agencyID",
"IDVID.PIID",
"IDVID.modNumber",
"lastDateToOrder",
"totalEstimatedOrderValue",
"whoCanUse",
"individualOrderLimit",
"typeOfIDC",
"multipleOrSingleAwardIDC",
"principalPlaceOfPerformance.locationCode",
"competitiveProcedures",
"smallBusinessCompetitivenessDemonstrationProgram",
"vendorLocation.streetAddress2",
"treasuryAccount.subAccountCode",
"CCRException",
"typeOfFeeForUseOfService",
"agencyID",
"modNumber",
"fiscalYear",
"nonGovernmentalDollars",
"typeOfAgreement",
"agencyIdentifier",
"mainAccountCode",
"nonTraditionalGovernmentContractorParticipation",
"PSCCode",
"isOtherbusinessOrOrganization",
"isUSGovernmentEntity",
"isArchitectureAndEngineering",
"isConstructionFirm",
"isResearchAndDevelopment",
"isServiceProvider",
"isAlaskanNativeServicingInstitution",
"isNativeHawaiianServicingInstitution",
"transactionNumber",
"subAccountCode",
"emailAddress",
"fixedFeeValue",
"orderingProcedure",
"Award Number",
"Project Title",
"Award Abstract",
"Anticipated Award Amount",
"DoD Awarding Office",
"Fiscal Year",
"Recipient Organization",
"PI First Name",
"PI Last Name",
"PI Type",
"Potential Period of Performance Start",
"Potential Period of Performance End",
"Creation Date",
"Modified Date",
"Contract Modification Number",
"IDV Type",
"Part 8 or Part 13",
"Multiple or Single Award IDC",
"Referenced IDV",
"Contracting Department",
"Contracting Department ID",
"PIID Agency",
"Agency ID",
"Contracting Office",
"Contracting Office ID",
"Last Date to Order",
"Ceiling Amount from IDV ($)",
"Dollars Obligated on Orders ($)",
"Ceiling Remaining ($)",
"nameNAICS",
"nameFullPSC",
"PSC Code",
"Website URL",
"POC Email",
"Requirement Description",
"Program Acronym",
"National Interest Action",
"Major Program",
"Fee for Use of Service",
"Max Order Limit ($)",
"Who Can Use",
"Contracting Officer's Business Size Determination",
"Socio Economic Data",
"tag",
"ACCT_NUM",
"ATA",
"AID",
"BPOA",
"EPOA",
"A",
"MAIN",
"SUB",
"GWA_TAS_NAME",
"Agency AID",
"FR Entity Type",
"FR Entity Description",
"financial_indicator_type2",
"FIN_IND_TYP2 description",
"Function Code",
"Function Description",
"Sub Function Code",
"Sub Function Description",
"DT_TM_ESTAB",
"DT_END",
"agencySpecificAwardElementsnonFederalFundingAmount",
"awardContractIDagencyID",
"awardContractIDPIID",
"awardContractIDmodNumber",
"awardContractIDtransactionNumber",
"referencedIDVIDagencyID",
"referencedIDVIDPIID",
"referencedIDVIDmodNumber",
"contractDataGFE-GFP",
"vendorHeadervendorName",
"vendorSocioEconomicIndicatorsisAlaskanNativeOwnedBusinessCorporationOrFirm",
"vendorSocioEconomicIndicatorsisAmericanIndianOwnedBusinessBusinessBusiness",
"vendorSocioEconomicIndicatorsisIndianTribe",
"vendorSocioEconomicIndicatorsisNativeHawaiianOwnedBusinessOrganizationOrFirm",
"vendorSocioEconomicIndicatorsisTriballyOwnedBusinessBusinessFirm",
"vendorSocioEconomicIndicatorsisSmallBusiness",
"vendorSocioEconomicIndicatorsisVeteranOwnedBusiness",
"vendorSocioEconomicIndicatorsisServiceDisabledVeteranOwnedBusinessBusiness",
"vendorSocioEconomicIndicatorsisWomanOwnedBusiness",
"minorityOwnedisMinorityOwnedBusiness",
"minorityOwnedisSubcontinentAsianIndianAmericanOwnedBusiness",
"minorityOwnedisAsianPacificAmericanOwnedBusiness",
"minorityOwnedisBlackAmericanOwnedBusiness",
"minorityOwnedisHispanicAmericanOwnedBusiness",
"minorityOwnedisNativeAmericanOwnedBusiness",
"minorityOwnedisOtherMinorityOwned",
"vendorSocioEconomicIndicatorsisVerySmallBusiness",
"vendorSocioEconomicIndicatorsisWomanOwnedBusinessSmallBusiness",
"vendorSocioEconomicIndicatorsisEconomicallyDisadvantagedWomanOwnedSmallBusiness",
"vendorSocioEconomicIndicatorsisJointVentureWomenOwnedSmallBusiness",
"vendorSocioEconomicIndicatorsisJointVentureWomenOwnedSmallBusiness",
"vendorBusinessTypesisCommunityDevelopedCorporationOwnedFirm",
"vendorBusinessTypesisLaborSurplusAreaFirm",
"federalGovernmentisFederalGovernment",
"federalGovernmentisFederallyFundedResearchAndDevelopmentCorp",
"federalGovernmentisFederalGovernmentAgency",
"vendorBusinessTypesisStateGovernment",
"localGovernmentisLocalGovernment",
"localGovernmentisCityLocalGovernment",
"localGovernmentisCountyLocalGovernment",
"localGovernmentisInterMunicipalLocalGovernment",
"localGovernmentisLocalGovernmentOwned",
"localGovernmentisMunicipalityLocalGovernment",
"localGovernmentisSchoolDistrictLocalGovernment",
"localGovernmentisTownshipLocalGovernment",
"vendorBusinessTypesisTribalGovernment",
"vendorBusinessTypesisForeignGovernment",
"businessOrOrganizationTypeisCorporateEntityNotTaxExempt",
"businessOrOrganizationTypeisCorporateEntityTaxExempt",
"businessOrOrganizationTypeisPartnershipOrLimitedLiabilityPartnership",
"businessOrOrganizationTypeisSolePropreitorship",
"businessOrOrganizationTypeisSmallAgriculturalCooperative",
"businessOrOrganizationTypeisInternationalOrganization",
"businessOrOrganizationTypeisUSGovernmentEntity",
"vendorLineOfBusinessisCommunityDevelopmentCorporation",
"vendorLineOfBusinessisDomesticShelter",
"vendorLineOfBusinessisEducationalInstitution",
"vendorLineOfBusinessisFoundation",
"vendorLineOfBusinessisHospital",
"vendorLineOfBusinessisManufacturerOfGoods",
"vendorLineOfBusinessisVeterinaryHospital",
"vendorLineOfBusinessisHispanicServicingInstitution",
"vendorRelationshipWithFederalGovernmentreceivesContracts",
"vendorRelationshipWithFederalGovernmentreceivesGrants",
"vendorRelationshipWithFederalGovernmentreceivesContractsAndGrants",
"typeOfGovernmentEntityisAirportAuthority",
"typeOfGovernmentEntityisCouncilOfGovernments",
"typeOfGovernmentEntityisHousingAuthoritiesPublicOrTribal",
"typeOfGovernmentEntityisInterstateEntity",
"typeOfGovernmentEntityisPlanningCommission",
"typeOfGovernmentEntityisPortAuthority",
"typeOfGovernmentEntityisTransitAuthority",
"vendorOrganizationFactorsisSubchapterSCorporation",
"vendorOrganizationFactorsisLimitedLiabilityCorporation",
"vendorOrganizationFactorsisForeignOwnedAndLocated",
"profitStructureisForProfitOrganization",
"profitStructureisNonprofitOrganization",
"profitStructureisOtherNotForProfitOrganization",
"vendorOrganizationFactorsisShelteredWorkshop",
"typeOfEducationalEntityis1862LandGrantCollege",
"typeOfEducationalEntityis1890LandGrantCollege",
"typeOfEducationalEntityis1994LandGrantCollege",
"typeOfEducationalEntityisHistoricallyBlackCollegeOrUniversity",
"typeOfEducationalEntityisMinorityInstitution",
"typeOfEducationalEntityisPrivateUniversityOrCollege",
"typeOfEducationalEntityisSchoolOfForestry",
"typeOfEducationalEntityisStateControlledInstitutionOfHigherLearning",
"typeOfEducationalEntityisTribalCollege",
"typeOfEducationalEntityisVeterinaryCollege",
"typeOfEducationalEntityisAlaskanNativeServicingInstitution",
"typeOfEducationalEntityisNativeHawaiianServicingInstitution",
"vendorCertificationsisDOTCertifiedDisadvantagedBusinessEnterprise",
"vendorCertificationsisSelfCertifiedSmallDisadvantagedBusiness",
"vendorCertificationsisSBACertifiedSmallDisadvantagedBusiness",
"vendorCertificationsisSBACertified8AProgramParticipant",
"vendorCertificationsisSelfCertifiedHUBZoneJointVenture",
"vendorCertificationsisSBACertifiedHUBZone",
"vendorCertificationsisSBACertified8AJointVenture",
"vendorLocationstreetAddress",
"vendorLocationcity",
"vendorLocationstate",
"vendorLocationZIPCode",
"vendorLocationcountryCode",
"vendorDUNSInformationDUNSNumber",
"vendorDUNSInformationvendorName",
"vendorDUNSInformationglobalParentDUNSNumber",
"vendorDUNSInformationglobalParentDUNSName",
"vendorcontractorName",
"vendorcontractingOfficerBusinessSizeDetermination",
"principalPlaceOfPerformancelocationCode",
"principalPlaceOfPerformancestateCode",
"principalPlaceOfPerformancecountryCode",
"treasuryAccountSymbolagencyIdentifier",
"treasuryAccountSymbolmainAccountCode",
"listOfAdditionalReportingValuesadditionalReportingValue",
"vendorOrganizationFactorsstateOfIncorporation",
"vendorOrganizationFactorscountryOfIncorporation",
"vendorOrganizationFactorsorganizationalType",
"vendorLocationphoneNo",
"vendorLocationfaxNo",
"vendorLocationcongressionalDistrictCode",
"vendorSiteDetailsvendorSiteCode",
"vendorSiteDetailsvendorAlternateSiteCode",
"vendorDUNSInformationcageCode",
"ccrRegistrationDetailsregistrationDate",
"ccrRegistrationDetailsrenewalDate",
"genericStringsgenericString01",
"treasuryAccountSymbolsubAccountCode",
"vendorHeadervendorAlternateName",
"vendorHeadervendorLegalOrganizationName",
"vendorHeadervendorDoingAsBusinessName",
"otherTransactionAwardContractIDagencyID",
"otherTransactionAwardContractIDPIID",
"otherTransactionAwardContractIDmodNumber",
"relevantContractDatesfiscalYear",
"dollarValuesnonGovernmentalDollars",
"totalDollarValuestotalNonGovernmentalDollars",
"contractDatatypeOfAgreement",
"contractDatanonTraditionalGovernmentContractorParticipation",
"contractDetailPSCCode",
"iDVIDagencyID",
"iDVIDPIID",
"iDVIDmodNumber",
"agencySpecificAwardElementsagencyID",
"pRNumber",
"closeoutPR",
"procurementPlacementCode",
"solicitationIssueDate",
"listOfContractAdministrationsDelegatedcontractAdministrationDelegated",
"advisoryOrAssistanceServicesContract",
"supportServicesTypeContract",
"newTechnologyOrPatentRightsClause",
"managementReportingRequirements",
"propertyFinancialReporting",
"valueEngineeringClause",
"securityCode",
"administratorCode",
"contractingOfficerCode",
"negotiatorCode",
"cOTRName",
"organizationCode",
"contractFundCode",
"isPhysicallyComplete",
"physicalCompletionDate",
"finalInvoicePaidDate",
"destroyDate",
"accountingInstallationNumber",
"principalInvestigatorfirstName",
"principalInvestigatorlastName",
"fieldOfScienceOrEngineering",
"installationUnique",
"offerorsProposalNumber",
"nonFederalFundingAmount",
"fundedThroughDate",
"cancellationDate",
"cFDAProgramIdentificationNumber",
"alternateCOTRName",
"vendorLocationstreetAddress2",
"alternatePrincipalInvestigatorfirstName",
"alternatePrincipalInvestigatorlastName",
"accessionNumber",
"principalInvestigatormiddleInitial",
"totalObligatedAmount",
"totalBaseAndExercisedOptionsValue",
"totalBaseAndAllOptionsValue",
"contractMarketingData",
"seaTransportation",
"treasuryAccountSymbol",
"claimantProgramCode",
"systemEquipmentCode",
"vendorHeader",
"isSmallBusiness",
"streetAddress",
"city",
"state",
"ZIPCode",
"countryCode",
"phoneNo",
"faxNo",
"congressionalDistrictCode",
"DUNSNumber",
"globalParentDUNSNumber",
"globalParentDUNSName",
"registrationDate",
"renewalDate",
"stateCode",
"preferencePrograms",
"approvedBy",
"approvedDate",
"referencedIDVPart8OrPart13",
"referencedIDVMultipleOrSingle",
"referencedIDVType",
"listOfAdditionalReportingValues",
"additionalReportingValue",
"cageCode",
"contractorName",
"contractID",
"websiteURL",
"locationCode",
"programAcronym",
"initiative",
"totalNonGovernmentalDollars",
"firstName",
"lastName",
"contractAdministrationDelegated",
"streetAddress2",
"middleInitial",
"agencySpecificIDVElements",
"NASASpecificIDVElements",
"isAlaskanNativeOwnedCorporationOrFirm",
"isAmericanIndianOwned",
"isNativeHawaiianOwnedOrganizationOrFirm",
"isTriballyOwnedFirm",
"isVeteranOwned",
"isServiceRelatedDisabledVeteranOwnedBusiness",
"isWomenOwned",
"isMinorityOwned",
"isSubContinentAsianAmericanOwnedBusiness",
"isOtherMinorityOwned",
"isWomenOwnedSmallBusiness",
"isEconomicallyDisadvantagedWomenOwnedSmallBusiness",
"isJointVentureEconomicallyDisadvantagedWomenOwnedSmallBusiness",
"isFederalGovernment",
"isFederalGovernmentAgency",
"isCityLocalGovernment",
"isCountyLocalGovernment",
"isLocalGovernmentOwned",
"isMunicipalityLocalGovernment",
"isCorporateEntityNotTaxExempt",
"isPartnershipOrLimitedLiabilityPartnership",
"isCommunityDevelopmentCorporation",
"modified",
"nameVendorDUNS",
"contractIDIDVIDagencyID",
"contractIDIDVIDPIID",
"contractIDIDVIDmodNumber",
"feeRangeLowerValue",
"feeRangeUpperValue",
"streetAddress3",
"divisionName",
"divisionNumberOrOfficeCode"
)
nameActual = c(
"idContractIDV",
"nameVendorParent",
"codeSubContractingPlan",
"typeSolicitationProcedure",
"typeAward",
"idAgencyAward",
"nameAgencyAward",
"idAward",
"idModAward",
"numberTransaction",
"datetimeContractSigned",
"datetimeContractEffective",
"datetimeContractCompletionCurrent",
"datetimeContractCompletionUltimate",
"amountObligation",
"amountBaseAndExercisedOption",
"amountBaseAllOption",
"amountObligationTotal",
"amountBaseAndExercisedOptionTotal",
"amountBaseAllOptionTotal",
"idAgencyPurchaser",
"nameAgencyPurchaser",
"idOfficePurchaser",
"nameOfficeAward",
"idAgencyFundingRequest",
"descriptionAgencyFundingRequest",
"idAgencyFunding",
"descriptionOfficeFundingRequest",
"codeDomesticForeignEntity",
"typeDomesticForeignEntity",
"amountMarketingDataFee",
"codeAction",
"typeAward",
"codeCostPricingData",
"typeCostPricingData",
"codeNationalInterestAction",
"nameNationalInterestAction",
"descriptionObligation",
"codeInherentlyGovernmentFunction",
"typeInherentlyGovernmentFunction",
"hasGFEOrGFP",
"typeGFEGFP",
"isSeaTransported",
"typeSeaTransportation",
"codeUndefinitizedAction",
"typeUndefinitizedAction",
"codeConsolidatedContract",
"typeConsolidatedContract",
"codePerformanceBasedServiceContract",
"typePerformanceBasedServiceContract",
"codeHumanitarianPeackeepingOperation",
"typeHumanitarianPeackeepingOperation",
"codeTreasuryAccounts",
"codeContractFinancing",
"typeContractFinancing",
"hasCreditCardPurchaseMethod",
"descriptionCreditCardPurchaseMethod",
"countActions",
"hasClingerCohenActPlanning",
"typeClingerCohenAct",
"codeMaterialsSuppliesArticlesEquipment",
"typeMaterialsSuppliesArticlesEquipment",
"hasLaborStandardsClause",
"typeLaborStandards",
"hasConstructionWageRequirements",
"typeConstructionWageRateRequirements",
"codeInterAgencyContractingAuthority",
"codeProductService",
"typeInterAgencyContractingAuthority",
"codeContractBundling",
"typeProductOrService",
"codeProgramClaimant",
"typeContractBundling",
"idNAICS",
"typeProgramClaimant",
"codeRecoveredMaterials",
"nameNAICS",
"codeManufacturingOrganization",
"typeRecoveredMaterials",
"codeEquipmentSystem",
"typeManufacturingOrganization",
"codeTechnologyItem",
"typeEquipmentSystem",
"codeEPADesignatedProduct",
"typeTechnologyItem",
"codeCountryOriginProductService",
"typeEPADesignatedProduct",
"codePlaceOfManufacture",
"countryProductServicePlaceOfManufacture",
"isAlaskanNativeOwnedBusiness",
"typeProductServicePlaceOfManufacture",
"isAmericanIndianOwnedBusinessBusinessBusiness",
"isIndianTribeOwned",
"isNativeHawaiianOwnedBusiness",
"isTriballyOwnedBusinessBusiness",
"isSmallBusiness",
"isVeteranOwnedBusiness",
"isServiceDisabledVeteranOwnedBusiness",
"isWomanOwnedBusiness",
"removeMinorityOwned",
"isVerySmallBusiness",
"isWomanOwnedBusinessSmallBusiness",
"isEconomicallyDisadvantagedWomanOwnedSmallBusiness",
"isJointVentureWomenOwnedSmallBusiness",
"isJointVentureWomenOwnedSmallBusiness",
"isCommunityDevelopedCorporationOwned",
"isLaborSurplusAreaFirm",
"removefederalGovernmentOwned",
"isStateGovernment",
"removeLocalGovernment",
"isTribalGovernment",
"isForeignGovernment",
"removeBusinessOrOrganizationType",
"isArchitectureAndEngineering",
"isCommunityDevelopedCorporationOwnedFirm",
"isConstructionFirm",
"isDomesticShelter",
"isEducationalInstitution",
"isFoundation",
"isHospital",
"isManufacturerOfGoods",
"isResearchAndDevelopment",
"isServiceProvider",
"isVeterinaryHospital",
"isHispanicServicingInstitution",
"hasContracts",
"hasGrants",
"hasContractsAndGrants",
"isAirportAuthority",
"isCouncilOfGovernments",
"isHousingAuthoritiesPublicOrTribal",
"isInterstateEntity",
"isPlanningCommission",
"isPortAuthority",
"isTransitAuthority",
"isSubchapterSCorporation",
"isLimitedLiabilityCorporation",
"isForeignOwnedAndLocated",
"removeVendorprofitStructure",
"isShelteredWorkshop",
"codeStateIncorporation",
"codeCountryIncorporation",
"nameStateIncorporation",
"typeOrganization",
"nameCountryIncorporation",
"is1862LandGrantCollege",
"is1890LandGrantCollege",
"is1994LandGrantCollege",
"isHistoricallyBlackCollegeOrUniversity",
"isMinorityInstitution",
"isPrivateUniversityOrCollege",
"isSchoolOfForestry",
"isStateControlledInstitutionOfHigherLearning",
"isTribalCollege",
"isVeterinaryCollege",
"isAlaskanNativeServicingInstitution",
"isNativeHawaiianServicingInstitution",
"isDOTCertifiedDisadvantagedBusinessEnterprise",
"isSelfCertifiedSmallDisadvantagedBusiness",
"isSBACertifiedSmallDisadvantagedBusiness",
"isSBACertified8AProgramParticipant",
"isSelfCertifiedHUBZoneJointVenture",
"isSBACertifiedHUBZone",
"isSBACertified8AJointVenture",
"addressStreet1Vendor",
"cityVendor",
"codeStateVendor",
"zipcodeVendor",
"removeBendorstateDescription",
"codeCountryVendor",
"zipcodeAreaVendor",
"telephoneVendor",
"removeVendorcountryCodeDescription",
"faxVendor",
"codeCongressionalDistrictVendor",
"remove_vendortext",
"slugDUNS",
"nameVendorDUNS",
"slugDUNSParent",
"nameVendorParent",
"datetimeRegistrationSAM",
"datetimeRenewalSAM",
"codeStatePerformance",
"codeCountryPerformance",
"zipcodePerformance",
"codeCongressionalDistrictPerformance",
"codeContractCompetition",
"codeSolicitationProcedures",
"typeContractCompetition",
"codeSetAside",
"typeSolicitationProcedures",
"codeEvaluatedPreference",
"typeSetAside",
"codeResearch",
"typeEvaluatedPreference",
"countOffersReceived",
"typeResearch",
"codeCommercialAcquisitionProcedures",
"isCommercialItemTestProgram",
"typeCommercialAcquisitionProcedures",
"hasA76FairActAction",
"typeCommercialItemTestProgram",
"isFBOSourced",
"typeA76Action",
"isLocalAreaSetAside",
"typeFBO",
"codeSubContractPlan",
"typeLocalAreaSetAside",
"transactionCreatedBy",
"typeSubContractPlan",
"datetimeTransactionCreated",
"transactionLastModifiedBy",
"datetimeTransactionLastModified",
"codeTransactionStatus",
"transactionApprovedBy",
"typeInformationStatus",
"datetimeTransactionApproved",
"isMinorityOwnedBusiness",
"isSubcontinentAsianIndianAmericanOwnedBusiness",
"isAsianPacificAmericanOwnedBusiness",
"isBlackAmericanOwnedBusiness",
"isHispanicAmericanOwnedBusiness",
"isNativeAmericanOwnedBusiness",
"isOtherMinorityOwnedBusiness",
"isUSFederalGovernment",
"isFederallyFundedResearchAndDevelopmentCorp",
"isFederalAgency",
"isLocalGovernment",
"isCityGovernment",
"isCountyGovernment",
"isInterMunicipalLocalGovernment",
"isLocalGovernment",
"isSchoolDistrictLocalGovernment",
"isSchoolDistrictLocalGovernment",
"isTownshipLocalGovernment",
"isCorporateEntityNonTaxExempt",
"isCorporateEntityTaxExempt",
"isPartnershipOrLLP",
"isSolePropreitorship",
"isSmallAgriculturalCooperative",
"isInternationalOrganization",
"isOtherBusiness",
"isUSGovernmentEntity",
"isForProfitOrganization",
"isNonprofitOrganization",
"isOtherNotForProfitOrganization",
"nameContract",
"urlFPDSContractAtom",
"datetimeContractModified",
"codeContractModification",
"typeContractModification",
"hasCostOrPricingData",
"typeCostOrPricing",
"codeCostAccountingClause",
"typeCostAccountingClause",
"isMultiYearContract",
"typeContractMultiYears",
"codePerformanceLocation",
"isSmallBusinessCompetitivenessDemonstrationProgram",
"pctPriceDifferencePriceEvaluation",
"idSolicitation",
"typeAdditionalReportingValues",
"cageVendor",
"codeSetAsideSource",
"typeSetAsideSource",
"codeOffersSource",
"typeOffersSource",
"isTransactionClosed",
"idAgencyAward",
"nameAgencyAward",
"idContract",
"codeModification",
"datetimeLastContractOrder",
"amountEstimatedTotal",
"urlGovernmentData",
"codeWhoCanUse",
"emailContract",
"amountIndividualOrderLimit",
"codeIDC",
"typeIDC",
"codeStatutoryAuthority",
"codeIDVMultipleOrSingle",
"typeAwardMultipleOrSingle",
"codeIDV",
"typeAward",
"hasFeeForUse",
"typeFeeForUse",
"nameMajorProgram",
"codeIDCContract",
"typeIDCContract",
"acronymProgram",
"amountFeeLowerBound",
"amountFeeUpperBound",
"typeOrderingProcedure",
"codeFairOpportunityException",
"typeFairOpportunityException",
"transactionClosedBy",
"datetimeTransactionClosed",
"codeNotCompeteReason",
"typeNotCompeteReason",
"codeIDVPart8OrPart13",
"addressStreet2Vendor",
"codeIDVSetAside",
"typeAwardSetAside",
"countOffersReceivedIDV",
"datetimeGenericStrings",
"idAgencyAwardSpecific",
"idPRNumber",
"hasPRCloseOut",
"codePlacementAgency",
"isAdvisoryOrAssistanceServicesContract",
"hasSupportServicesContract",
"hasNewTechnologyOrPatentRightsClause",
"hasManagementReportingRequirements",
"hasPropertyFinancialReporting",
"hasEngineeringClause",
"hasSecurityCode",
"codeAdministrator",
"codeContractingOfficer",
"codeNegotiator",
"nameCOTR",
"codeAwardOrganization",
"codeContractFund",
"isPhysicallyComplete",
"idAccountingInstallation",
"datetimeFundedThroughDate",
"codePrincipalInvestigator",
"idScienceOrEngineering",
"datetimeCompletionPhysical",
"idAccession",
"datetimeDestroy",
"datetimeSolicitationIssued",
"codeContractAdministrationsDelegated",
"datetimeFinalInvoicePaid",
"amountMarketingDataFeeFixed",
"isWalshHealyAct",
"removeWalshHealyActDescription",
"isMandatedServiceContractAct",
"removeMandatesserviceContractActDescription",
"isDavisBaconAct",
"removeMandatesDavisBaconActDescription",
"idContract",
"idContractIDV",
"idAgency",
"codeModification",
"idAgencyAward",
"nameAgencyAward",
"idOfficeAward",
"nameOfficeAward",
"idAgencyFunding",
"idOfficeFunding",
"nameOfficeFunding",
"codeAgency",
"nameAgency",
"idDepartment",
"nameDepartment",
"dateLastModified",
"lastModifiedBy",
"dateAwardCompletion",
"dateCreated",
"dateSigned",
"dateEffective",
"dateCompletion",
"amountObligation",
"amountContract",
"typeCostPricingData",
"statusAward",
"typeContract",
"createdBy",
"descriptionRequirement",
"modificationReason",
"legislativeMandates",
"localAreaSetAside",
"indicatorsSocioEconomic",
"isMultiYearContract",
"codeNationalInterestAction",
"nameNationalInterestAction",
"idNAICS",
"nameNAICS",
"codeProductService",
"descriptionProductService",
"codeCongressionalDistrictPerformance",
"countryPerformance",
"nameStatePerformance",
"cityVendor",
"codeCongressionalDistrictVendor",
"codeCountryVendor",
"nameCountryVendor",
"idDUNSVendor",
"nameVendorDoingBusinessAs",
"nameVendor",
"codeStateVendor",
"nameStateVendor",
"zipcodeVendor",
"nameVendorGlobal",
"codeTreasuryAccountSymbol",
"codeInstallnentAward",
"idContract",
"idContractIDV",
"codeModification",
"numberTransaction",
"typeAward",
"amountObligation",
"dateContractSigned",
"dateSolicitation",
"idAgencyAward",
"nameAgencyAward",
"nameOfficeAward",
"typeProductOrService",
"codeProductService",
"descriptionProductService",
"idNAICS",
"nameNAICS",
"slugDUNS",
"nameVendor",
"cityVendor",
"codeStateVendor",
"zipcodeVendor",
"slugDUNSParent",
"nameVendorParent",
"codeAdditionalReporting",
"descriptionAdditonalReporting",
"idAgency",
"idDepartment",
"nameAgency",
"slugAgency",
"isBrokerRegistered",
"isASPRegistered",
"sourceOffice",
"idOffice",
"nameOffice",
"slugOffice",
"nameAdminister",
"idAdminister",
"codeFREC",
"descriptionFREC",
"sourceOfficeCode",
"isFPDSOffice",
"isASPRegisteredOld",
"nameAgencyOld",
"nameOfficeOld",
"isFREC",
"descriptionMission",
"url",
"icon",
"idAgency",
"nameAgency",
"idBureau",
"nameBureau",
"idAccount",
"nameAccount",
"idTreasuryAccount",
"idFunction",
"nameFunction",
"typeCategoryBEA",
"typeBudget",
"fy1976",
"fyTQ",
"fy1977",
"fy1978",
"fy1979",
"fy1980",
"fy1981",
"fy1982",
"fy1983",
"fy1984",
"fy1985",
"fy1986",
"fy1987",
"fy1988",
"fy1989",
"fy1990",
"fy1991",
"fy1992",
"fy1993",
"fy1994",
"fy1995",
"fy1996",
"fy1997",
"fy1998",
"fy1999",
"fy2000",
"fy2001",
"fy2002",
"fy2003",
"fy2004",
"fy2005",
"fy2006",
"fy2007",
"fy2008",
"fy2009",
"fy2010",
"fy2011",
"fy2012",
"fy2013",
"fy2014",
"fy2015",
"fy2016",
"fy2017",
"fy2018",
"fy2019",
"fy2020",
"fy2021",
"fy2022",
"nameCountryPerformance",
"idDUNSParent",
"codeResearch",
"nameAgency",
"idAgency",
"typeDescriptionManufacter",
"codeCountryOriginProductService",
"idContract",
"dateCompletion",
"idNAICS",
"codeProductService",
"descriptionProductService",
"countryVendor",
"idDUNSVendor",
"typeContract",
"typeAward",
"amountContract",
"amountObligation",
"nameSolicitation",
"idSolicitation",
"nameAgency",
"nameOffice",
"detailsOffice",
"typeSolicitation",
"typeSolicitationOriginal",
"datePosted",
"datePostedOriginal",
"descriptionDeadLine",
"policyArchiving",
"datetimeArchive",
"typeSetAside",
"codeClassification",
"idNAICS",
"addressContractingOffice",
"descriptionContractingPointContact",
"locationContractPerformance",
"detailsAdditionalLinks",
"descriptionSolicitation",
"isActive",
"urlSolicitation",
"idAward",
"datePosted",
"dateAward",
"amountAward",
"descriptionLineNumber",
"nameAwardeeAddress",
"numberItem",
"dateSolicitation",
"nameAgency",
"nameOffice",
"detailsOffice",
"zipcodeAgency",
"codeClassification",
"locationOffice",
"nameSolicitation",
"idSolicitation",
"dateResponse",
"descriptionContractingPointContact",
"descriptionSolicitation",
"urlSolicitation",
"codeCountryPerformance",
"zipcodePerformace",
"locationContractPerformance",
"detailsSolicitation",
"dateArchive",
"emailContact",
"typeSetAside",
"typeAward",
"idAward",
"amountAward",
"dateAward",
"nameAwardeeAddress",
"descriptionLineNumber",
"requirementsBureau",
"nameCompany",
"typeFAR",
"codeModification",
"descriptionFollowOn",
"idDONBR",
"emailContact",
"hasCorrections",
"urlSolicitation",
"isRecoveryAct",
"descriptionDocumentPackages",
"descriptionChanges",
"isWomanOwnedBusiness",
"isWomanOwnedBusinessSmallBusiness",
"idAgency",
"idContractIDV",
"urlCongressionalJustification",
"idAgencyAward",
"idContract",
"codeModification",
"numberTransaction",
"idOfficeIDV",
"idContractIDV",
"codeModIDV",
"datetimeContractSigned",
"datetimeContractEffective",
"datetimeContractCompletionCurrent",
"datetimeContractCompletionUltimate",
"amountObligation",
"amountBaseAndExercisedOption",
"amountBaseAllOption",
"idAgencyAward",
"idOfficeAward",
"idAgencyFunding",
"idOfficeFunding",
"codeDomesticForeignEntity",
"amountFeePaidForService",
"codeAction",
"codeCostPricingData",
"codeContractModification",
"nameMajorProgram",
"codeNationalInterestAction",
"descriptionObligation",
"codeInherentlyGovernmentFunction",
"hasGFEOrGFP",
"codeUndefinitizedAction",
"codeConsolidatedContract",
"codePerformanceBasedServiceContract",
"codeHumanitarianPeackeepingOperation",
"idCGAC",
"codeAccountOMB",
"amountObligationTreasury",
"hasCreditCardPurchaseMethod",
"countActions",
"hasClingerCohenActPlanning",
"codeMaterialsSuppliesArticlesEquipment",
"hasLaborStandardsClause",
"hasConstructionWageRequirements",
"codeInterAgencyContractingAuthority",
"codeProductService",
"codeContractBundling",
"idNAICS",
"codeRecoveredMaterials",
"codeManufacturingOrganization",
"codeEPADesignatedProduct",
"codeCountryOriginProductService",
"codePlaceOfManufacture",
"nameVendor",
"isAlaskanNativeOwnedBusinessCorporationOrFirm",
"isAmericanIndianOwnedBusinessBusinessBusiness",
"isIndianTribeOwned",
"isNativeHawaiianOwnedBusiness",
"isTriballyOwnedBusinessBusiness",
"isVeteranOwnedBusiness",
"isServiceDisabledVeteranOwnedBusiness",
"isWomanOwnedBusiness",
"isMinorityOwnedBusiness",
"isSubcontinentAsianIndianAmericanOwnedBusiness",
"isAsianPacificAmericanOwnedBusiness",
"isBlackAmericanOwnedBusiness",
"isHispanicAmericanOwnedBusiness",
"isNativeAmericanOwnedBusiness",
"isOtherMinorityOwnedBusiness",
"isVerySmallBusiness",
"isWomanOwnedSmallBusiness",
"isEconomicallyDisadvantagedWomanOwnedSmallBusiness",
"isJointVentureWomenOwnedSmallBusiness",
"isJointVentureWomenOwnedSmallBusiness",
"isCommunityDevelopedCorporationOwned",
"isLaborSurplusAreaFirm",
"isUSFederalGovernment",
"isFederallyFundedResearchAndDevelopmentCorp",
"isFederalAgency",
"isStateGovernment",
"isLocalGovernment",
"isCityGovernment",
"isCountyGovernment",
"isInterMunicipalLocalGovernment",
"isLocalGovernment",
"isSchoolDistrictLocalGovernment",
"isSchoolDistrictLocalGovernment",
"isTownshipLocalGovernment",
"isTribalGovernment",
"isForeignGovernment",
"isCorporateEntityNonTaxExempt",
"isCorporateEntityTaxExempt",
"isPartnershipOrLLP",
"isSolePropreitorship",
"isSmallAgriculturalCooperative",
"isInternationalOrganization",
"isCommunityDevelopedCorporationOwnedFirm",
"isDomesticShelter",
"isEducationalInstitution",
"isFoundation",
"isHospital",
"isManufacturerOfGoods",
"isVeterinaryHospital",
"isHispanicServicingInstitution",
"hasContracts",
"hasGrants",
"hasContractsAndGrants",
"isAirportAuthority",
"isCouncilOfGovernments",
"isHousingAuthoritiesPublicOrTribal",
"isInterstateEntity",
"isPlanningCommission",
"isPortAuthority",
"isTransitAuthority",
"isSubchapterSCorporation",
"isLimitedLiabilityCorporation",
"isForeignOwnedAndLocated",
"isForProfitOrganization",
"isNonprofitOrganization",
"isOtherNotForProfitOrganization",
"isShelteredWorkshop",
"codeStateIncorporation",
"codeCountryIncorporation",
"typeOrganization",
"is1862LandGrantCollege",
"is1890LandGrantCollege",
"is1994LandGrantCollege",
"isHistoricallyBlackCollegeOrUniversity",
"isMinorityInstitution",
"isPrivateUniversityOrCollege",
"isSchoolOfForestry",
"isStateControlledInstitutionOfHigherLearning",
"isTribalCollege",
"isVeterinaryCollege",
"isDOTCertifiedDisadvantagedBusinessEnterprise",
"isSelfCertifiedSmallDisadvantagedBusiness",
"isSBACertifiedSmallDisadvantagedBusiness",
"isSBACertified8AProgramParticipant",
"isSelfCertifiedHUBZoneJointVenture",
"isSBACertifiedHUBZone",
"isSBACertified8AJointVenture",
"addressStreet1Vendor",
"cityVendor",
"codeStateVendor",
"zipcodeVendor",
"codeCountryVendor",
"telephoneVendor",
"faxVendor",
"codeCongressionalDistrictVendor",
"codeSiteVendor",
"codeSiteAlternateVendor",
"slugDUNS",
"cageVendor",
"datetimeRegistrationSAM",
"datetimeRenewalSAM",
"codeContractingOfficerBusinessSizeDetermination",
"codeStatePerformance",
"codeCountryPerformance",
"zipcodePerformance",
"codeCongressionalDistrictPerformance",
"codeContractCompetition",
"codeSolicitationProcedures",
"codeSetAside",
"codeSetAsideSource",
"codeEvaluatedPreference",
"countOffersReceived",
"codeOffersSource",
"codeCommercialAcquisitionProcedures",
"hasA76FairActAction",
"isLocalAreaSetAside",
"transactionCreatedBy",
"datetimeTransactionCreated",
"transactionLastModifiedBy",
"datetimeTransactionLastModified",
"transactionClosedBy",
"datetimeTransactionClosed",
"isTransactionClosed",
"codeStatusTransaction",
"codeAdditionalReporting",
"nameVendorLegal",
"codeIDVSetAside",
"codeFairOpportunityException",
"countOffersReceivedIDV",
"isCommercialItemTestProgram",
"isFBOSourced",
"nameVendorAlternate",
"hasCostOrPricingData",
"isMultiYearContract",
"codeNotCompeteReason",
"codeSubContractPlan",
"idSolicitation",
"codeContractFinancing",
"nameVendorDoingBusinessAs",
"codeTechnologyItem",
"codeCostAccountingClause",
"pctPriceDifferencePriceEvaluation",
"codeStatutoryAuthority",
"idAgencyAward",
"idContract",
"codeModification",
"dateLastOrder",
"amountEstimatedTotal",
"codeWhoCanUse",
"amountIndividualOrderLimit",
"codeIDCContract",
"codeIDC",
"codePerformanceLocation",
"codeCompetitiveProcedures",
"hasSmallBusinessCompetitivenessDemonstrationProgram",
"addressStreet2Vendor",
"codeTreasurySubAccountSymbol",
"idCCRException",
"codeFeeForUseOfService",
"idAgencyAward",
"codeModification",
"yearFiscalContract",
"amountNonGovernmentalDollars",
"typeAgreement",
"idCGAC",
"codeAccountOMB",
"codeNonTraditionalGovernmentContractorParticipation",
"codeProductService",
"isOtherBusiness",
"isUSGovernmentEntity",
"isArchitectureAndEngineering",
"isConstructionFirm",
"isResearchAndDevelopment",
"isServiceProvider",
"isAlaskanNativeServicingInstitution",
"isNativeHawaiianServicingInstitution",
"numberTransaction",
"codeTreasurySubAccountSymbol",
"emailContract",
"amountMarketingDataFeeFixed",
"typeOrderingProcedure",
"idAward",
"nameAward",
"descriptionAward",
"amountAward",
"nameOffice",
"yearFiscal",
"nameOrganization",
"nameFirstPrincipalInvestigator",
"nameLastPrincipalInvestigator",
"typePrincipalInvestigator",
"dateAwardStart",
"dateAwardEnd",
"datetimeCreated",
"datetimeModified",
"idModification",
"typeIDV",
"typePart8Part13",
"typeIDC",
"idContractIDV",
"nameDepartment",
"idDepartment",
"nameAgencyPIID",
"idAgency",
"nameOfficeAward",
"codeOfficePurchaser",
"dateOrderLast",
"amountCeilingIDV",
"amountObligation",
"amountRemaining",
"nameNAICS",
"nameFullPSC",
"codePSC",
"urlProgram",
"emailContact",
"descriptionRequirement",
"codeProgram",
"codeNationalInterestAction",
"nameMajorProgram",
"typeFeeForService",
"amountOrderMaximum",
"descriptionWhoCanUse",
"typeContractingOfficerBusinessSize",
"descriptionSetAsides",
"tag",
"idAccount",
"idATA",
"idAID",
"idBPOA",
"idEPOA",
"slugA",
"idMainAccount",
"idSubAccount",
"nameAccount",
"idAgency",
"slugOffice",
"nameOffice",
"slugFinancialIndicator",
"nameFinancialIndicator",
"idFunction",
"descriptionFunction",
"idSubFunction",
"descriptionSubFunction",
"datetimeEstablished",
"datetimeEnd",
"hasFundingNonFederal",
"idAgency",
"idContract",
"codeModification",
"transactionNumber",
"idAgencyIDV",
"idContractIDV",
"codeModIDV",
"hasGFEOrGFP",
"nameVendor",
"isAlaskanNativeOwnedBusinessCorporationOrFirm",
"isAmericanIndianOwnedBusinessBusinessBusiness",
"isIndianTribe",
"isNativeHawaiianOwnedBusinessOrganizationOrFirm",
"isTriballyOwnedBusinessBusinessFirm",
"isSmallBusiness",
"isVeteranOwnedBusiness",
"isServiceDisabledVeteranOwnedBusinessBusiness",
"isWomanOwnedBusiness",
"isMinorityOwnedBusiness",
"isSubcontinentAsianIndianAmericanOwnedBusiness",
"isAsianPacificAmericanOwnedBusiness",
"isBlackAmericanOwnedBusiness",
"isHispanicAmericanOwnedBusiness",
"isNativeAmericanOwnedBusiness",
"isOtherMinorityOwnedBusiness",
"isVerySmallBusiness",
"isWomanOwnedBusinessSmallBusiness",
"isEconomicallyDisadvantagedWomanOwnedSmallBusiness",
"isJointVentureWomenOwnedSmallBusiness",
"isJointVentureWomenOwnedSmallBusiness",
"isCommunityDevelopedCorporationOwnedFirm",
"isLaborSurplusAreaFirm",
"isUSFederalGovernment",
"isFederallyFundedResearchAndDevelopmentCorp",
"isFederalAgency",
"isStateGovernment",
"isLocalGovernment",
"isCityGovernment",
"isCountyGovernment",
"isInterMunicipalLocalGovernment",
"isLocalGovernment",
"isSchoolDistrictLocalGovernment",
"isSchoolDistrictLocalGovernment",
"isTownshipLocalGovernment",
"isTribalGovernment",
"isForeignGovernment",
"isCorporateEntityNonTaxExempt",
"isCorporateEntityTaxExempt",
"isPartnershipOrLLP",
"isSolePropreitorship",
"isSmallAgriculturalCooperative",
"isInternationalOrganization",
"isUSGovernmentEntity",
"isCommunityDevelopedCorporationOwnedFirm",
"isDomesticShelter",
"isEducationalInstitution",
"isFoundation",
"isHospital",
"isManufacturerOfGoods",
"isVeterinaryHospital",
"isHispanicServicingInstitution",
"hasFederalContracts",
"hasFederalGrants",
"hasFederalContractsAndGrants",
"isAirportAuthority",
"isCouncilOfGovernments",
"isHousingAuthoritiesPublicOrTribal",
"isInterstateEntity",
"isPlanningCommission",
"isPortAuthority",
"isTransitAuthority",
"isSubchapterSCorporation",
"isLimitedLiabilityCorporation",
"isForeignOwnedAndLocated",
"isForProfitOrganization",
"isNonprofitOrganization",
"isOtherNotForProfitOrganization",
"isShelteredWorkshop",
"is1862LandGrantCollege",
"is1890LandGrantCollege",
"is1994LandGrantCollege",
"isHistoricallyBlackCollegeOrUniversity",
"isMinorityInstitution",
"isPrivateUniversityOrCollege",
"isSchoolOfForestry",
"isStateControlledInstitutionOfHigherLearning",
"isTribalCollege",
"isVeterinaryCollege",
"isAlaskanNativeServicingInstitution",
"isNativeHawaiianServicingInstitution",
"isDOTCertifiedDisadvantagedBusinessEnterprise",
"isSelfCertifiedSmallDisadvantagedBusiness",
"isSBACertifiedSmallDisadvantagedBusiness",
"isSBACertified8AProgramParticipant",
"isSelfCertifiedHUBZoneJointVenture",
"isSBACertifiedHUBZone",
"isSBACertified8AJointVenture",
"vendorLocationstreetAddress",
"vendorLocationcity",
"vendorLocationstate",
"vendorLocationZIPCode",
"vendorLocationcountryCode",
"slugDUNS",
"nameVendor",
"slugDUNSParent",
"nameVendorParent",
"nameContractorVendor",
"codeContractingOfficerBusinessSizeDetermination",
"codePerformanceLocation",
"codeStatePerformance",
"codeCountryPerformance",
"idCGACAccount",
"codeAccountTreasury",
"codeAdditionalReporting",
"codeStateIncorporation",
"codeCountryIncorporation",
"typeOrganization",
"telephoneVendor",
"faxVendor",
"codeCongressionalDistrictVendor",
"codeSiteVendor",
"codeSiteAlternateVendor",
"cageVendor",
"datetimeRegistrationSAM",
"datetimeRenewalSAM",
"datetimeGeneric",
"codeSubAccountTreasury",
"nameVendorAlternate",
"nameVendorLegal",
"nameVendorDBA",
"idAgencyOtherTransactions",
"idContractOtherTransactions",
"codeModOtherTransactions",
"yearFiscalContract",
"amountObligationNonGovernment",
"amountTotalObligationNonGovernment",
"typeAgreement",
"codeNonTraditionalParticipation",
"codeProductService",
"idAgencyIDV",
"idContractIDV",
"codeModIDV",
"idAgency",
"numberPR",
"isCloseoutPR",
"codeProcurementPlacement",
"datetimeSolicitation",
"codeContractAdministrationDelegation",
"isAdvisoryOrAssistanceServicesContract",
"isSupportServicesContract",
"hasNewTechnologyOrPatentRightsClause",
"hasManagementReportingRequirements",
"hasPropertyFinancialReporting",
"hasValueEngineeringClause",
"hasSecurityCode",
"codeAdministrator",
"codeContractingOfficer",
"codeNegotiator",
"nameCOTRSignatory",
"codeOrganization",
"codeContractFund",
"isPhysicallyComplete",
"datetimePhysicalCompletion",
"datetimeFinalInvoicePaid",
"datetimeDestroy",
"numberAccountingInstallation",
"nameFirstPrincipalInvestigator",
"nameLastPrincipalInvestigator",
"numberFieldOfScienceOrEngineering",
"codeInstallationUnique",
"numberOfferorsProposal",
"amountFundingNonFederal",
"datetimeFundedThrough",
"datetimeCanceled",
"codeProgramFDA",
"nameCOTRAlternate",
"vendorLocationstreetAddress2",
"nameFirstPrincipalInvestigatorAlternate",
"nameLastPrincipalInvestigatorAlternate",
"idAccession",
"nameMiddlePrincipalInvestigator",
"amountObligationTotal",
"amountBaseAndExercisedOptionTotal",
"amountBaseAllOptionTotal",
"amountMarketingDataFee",
"isSeaTransported",
"codeTreasuryAccountSymbol",
"codeProgramClaimant",
"codeEquipmentSystem",
"nameVendorHeader",
"isSmallBusiness",
"addressStreet1",
"city",
"codeState",
"zipcode",
"codeCountry",
"telephone",
"fax",
"codeCongressionalDistrict",
"slugDUNS",
"slugDUNSParent",
"nameVendorParent",
"datetimeRegistrationSAM",
"datetimeRenewalSAM",
"codeState",
"codeSubContractPlan",
"transactionApprovedBy",
"datetimeTransactionApproved",
"codePart8OrPart13IDV",
"codeMultipleOrSingleIDV",
"codeReferencedIDV",
"codeListOfAdditionalReportingValues",
"codeAdditionalReportingValue",
"cageVendor",
"nameContractor",
"idContract",
"urlWebsite",
"location",
"codeProgram",
"idInitiative",
"amountNonGovernmentalDollars",
"nameFirst",
"nameLast",
"codeContractAdministrationDelegated",
"addressStreet2",
"initialMiddle",
"codeIDVElementsAgencySpecific",
"codeIDVElementsNASA",
"isAlaskanNativeOwnedCorporationOrFirm",
"isAmericanIndianOwned",
"isNativeHawaiianOwnedOrganizationOrFirm",
"isTriballyOwnedFirm",
"isVeteranOwned",
"isServiceRelatedDisabledVeteranOwnedBusiness",
"isWomenOwned",
"isMinorityOwned",
"isSubContinentAsianAmericanOwnedBusiness",
"isOtherMinorityOwned",
"isWomenOwnedSmallBusiness",
"isEconomicallyDisadvantagedWomenOwnedSmallBusiness",
"isJointVentureEconomicallyDisadvantagedWomenOwnedSmallBusiness",
"isFederalGovernment",
"isFederalGovernmentAgency",
"isCityLocalGovernment",
"isCountyLocalGovernment",
"isLocalGovernmentOwned",
"isMunicipalityLocalGovernment",
"isCorporateEntityNotTaxExempt",
"isPartnershipOrLimitedLiabilityPartnership",
"isCommunityDevelopmentCorporation",
"datetimeModified",
"nameVendorDUNS",
"idAgencyIDV2",
"idContractIDV2",
"codeModIDV2",
"amountfeeRangeLowerValue",
"amountfeeRangeUpperValue",
"addressStreet3",
"nameDivision",
"codeDivision"
)
tibble(nameFPDS, nameActual)
})
# munge -------------------------------------------------------------------
.pad_fdps_duns <-
function(duns = 3457,
total_characters = 9,
return_only_slug = T) {
zeros <- total_characters - nchar(duns)
if (zeros == 0) {
return(tibble(idDUNS = duns, slugDUNS = as.character(duns)))
}
zero_slug <- rep(0, zeros) %>% str_c(collapse = "")
slugDUNS <- glue("{zero_slug}{duns}") %>% as.character()
if (return_only_slug) {
return(slugDUNS)
}
tibble(idDUNS = duns, slugDUNS)
}
#' Pad DUNS for FDPS search
#'
#' @param duns vector of DUNS
#'
#' @return
#' @export
#'
#' @examples
pad_fdps_duns <-
function(duns = NULL) {
if (length(duns) == 0) {
stop("Enter DUNS")
}
duns %>%
map_dfr(function(duns) {
.pad_fdps_duns(duns = duns, return_only_slug = F)
})
}
.munge_fdps_data <-
function(data) {
data %>% .munge_data(clean_address = F)
}
.munge_fpds_names <-
function(data) {
dict_names <- dictionary_fpds_names()
fdps_names <-
names(data)
actual_names <-
fdps_names %>%
map_chr(function(x) {
df_row <-
dict_names %>%
mutate(nameFPDS = str_to_lower(nameFPDS)) %>%
filter(nameFPDS == str_to_lower(x)) %>%
distinct() %>%
slice(1)
if (nrow(df_row) == 0) {
glue::glue("Missing {x}") %>% message()
return(x)
}
df_row$nameActual
})
data %>%
set_names(actual_names)
}
.sort_slug <-
function(sort_method = NULL,
sort_descending = T) {
if (length(sort_method) == 0) {
return("")
}
sort_dir <- case_when(sort_descending ~ "desc=Y",
TRUE ~ "desc=N")
sort <-
sort_method %>% str_to_lower()
sort_slug <-
case_when(
sort %>% str_detect("amount") ~ glue("&sortBy=OBLIGATED_AMOUNT&{sort_dir}") %>% as.character(),
sort %>% str_detect("agency code") ~ glue("&sortBy=AGENCY_CODE&{sort_dir}") %>% as.character(),
sort %>% str_detect("agency name") ~ glue("sortBy=AGENCY_FULL_NAME&{sort_dir}") %>% as.character(),
sort %>% str_detect("date") ~ glue("sortBy=SIGNED_DATE&{sort_dir}") %>% as.character(),
sort %>% str_detect("contracing agency id") ~ glue("sortBy=CONTRACTING_AGENCY_ID&{sort_dir}") %>% as.character(),
sort %>% str_detect(
"contract agency name|contracting agency name|contracting agency full name"
) ~ glue("sortBy=CONTRACTING_AGENCY_NAME&{sort_dir}") %>% as.character(),
sort %>% str_detect("department name") ~ glue("sortBy=DEPARTMENT_FULL_NAME&{sort_dir}") %>% as.character(),
sort %>% str_detect("naics") ~ glue("sortBy=PRINCIPAL_NAICS_CODE&{sort_dir}") %>% as.character(),
sort %>% str_detect("product code|service code") ~ glue("sortBy=PRODUCT_OR_SERVICE_CODE&{sort_dir}") %>% as.character(),
sort %>% str_detect("vendor state") ~ glue("sortBy=VENDOR_ADDRESS_STATE_CODE&{sort_dir}") %>% as.character(),
sort %>% str_detect("vendor zip") ~ glue("sortBy=VENDOR_ADDRESS_ZIP_CODE&{sort_dir}") %>% as.character(),
sort %>% str_detect("performance country") ~ glue("sortBy=POP_COUNTRY_NAME&{sort_dir}") %>% as.character(),
sort %>% str_detect("performance state") ~ glue("sortBy=POP_STATE_NAME&{sort_dir}") %>% as.character(),
sort %>% str_detect("local area") ~ glue("sortBy=LOCAL_AREA_SET_ASIDE&{sort_dir}") %>% as.character(),
sort %>% str_detect("treasury account") ~ glue("sortBy=TREASURY_ACCOUNT_SYMBOL&{sort_dir}") %>% as.character(),
sort %>% str_detect("local area") ~ glue("sortBy=LOCAL_AREA_SET_ASIDE&{sort_dir}") %>% as.character(),
sort %>% str_detect("non government") ~ glue("sortBy=NON_GOVERNMENT_VALUE&{sort_dir}") %>% as.character(),
sort %>% str_detect("contract fiscal year") ~ glue("sortBy=CONTRACT_FISCAL_YEAR&{sort_dir}") %>% as.character(),
TRUE ~ ""
)
sort_slug
}
.create_value_slug <-
function(values = "$100,000,000 | $200,000,000", escape = F) {
options(scipen = 999999)
values <- as.character(values)
values <-
values %>% str_split("\\|") %>% flatten_chr() %>% str_trim() %>% map_dbl(readr::parse_number) %>%
str_c(collapse = ",")
value_slug <- glue("[{values}]") %>% as.character()
if (escape) {
value_slug <- url_escape(value_slug)
}
value_slug
}
.shorten_dates <- function(dates = c("2018-01-01", "2019-03-01")) {
if (length(dates) == 1) {
dates <- c(dates, dates)
}
as.character(dates) %>% readr::parse_date() %>% as.character() %>% str_replace_all("\\-", "/") %>%
str_c(collapse = " | ")
}
.shorten_amounts <-
function(amounts = c(1000000, 20000000)) {
if (length(amounts) == 1) {
amounts <- c(amounts, amounts)
}
amounts %>% as.character() %>% map_dbl(readr::parse_number) %>% formattable::currency(digits = 0) %>%
as.character() %>%
str_c(collapse = " | ")
}
.shorten_count <-
function(amounts = c(10, 20)) {
if (length(amounts) == 1) {
amounts <- c(amounts, amounts)
}
amounts %>% as.character() %>% map_dbl(readr::parse_number) %>%
as.character() %>%
str_c(collapse = " | ")
}
.shorten_character <-
function(x = c("A", "B")) {
x %>%
str_c(collapse = " | ")
}
.count_results <-
function(url = "https://www.fpds.gov/ezsearch/fpdsportal?s=FPDS&q=%20AWARD_COMPLETION_DATE%3A%5B2019/04/02%2C2019/04/03%5D&indexName=awardfull&y=0&x=0&templateName=1.4") {
page <- url %>% read_html()
page %>% html_nodes("b~ b+ b") %>% html_text() %>% readr::parse_number()
}
.create_date_slug <-
function(dates = "2019/04/01 | 2019/04/03", escape = F) {
values <- as.character(dates)
values <-
values %>%
str_replace_all("\\-", "/") %>%
str_split("\\|") %>% flatten_chr() %>% str_trim() %>%
str_c(collapse = ",")
if (!dates %>% str_detect("|")) {
values <- str_c(values, values, sep = ",")
}
value_slug <- glue("[{values}]") %>% as.character()
if (escape) {
value_slug <- url_escape(value_slug)
}
value_slug
}
# list_munge --------------------------------------------------------------
.munge_double_list <-
function(result_list,
date_columns = c(
"VENDOR_RENEWAL_DATE",
"SOLICITATION_DATE",
"FINAL_INVOICE_PAID_DATE",
"SYS_LAST_MODIFIED_DATE",
"SIGNED_DATE",
"FUNDED_THROUGH_DATE",
"ESTIMATED_COMPLETION_DATE",
"LAST_MODIFIED_DATE",
"EFFECTIVE_DATE",
"CANCELLATION_DATE",
"LAST_DATE_TO_ORDER",
"PHYSICAL_COMPLETION_DATE",
"SOLICITATION_ISSUE_DATE",
"CREATED_DATE",
"DESTROY_DATE",
"AWARD_COMPLETION_DATE",
"LAST_MOD_DATE",
"VENDOR_REGISTRATION_DATE",
"REVEAL_DATE"
)
,
amount_columns = c(
"ANNUAL_REVENUE",
"BASE_EXERCISED_OPTIONS_VALUE",
"CURRENT_CONTRACT_VALUE",
"DOLLARS_OBLIGATED",
"CONTRACT_VALUE",
"FEE_RANGE_LOWER_VALUE",
"FEE_RANGE_UPPER_VALUE",
"FIXED_FEE_VALUE",
"NON_GOVERNMENT_VALUE",
"OBLIGATED_AMOUNT",
"TOTAL_CURRENT_CONTRACT_VALUE",
"TOTAL_DOLLARS_OBLIGATED",
"TOTAL_NON_GOVERNMENT_VALUE",
"TOTAL_ULTIMATE_CONTRACT_VALUE",
"ULTIMATE_CONTRACT_VALUE"
),
count_columns = c("NUMBER_OF_OFFERS_RECEIVED", "NUMBER_OF_EMPLOYEES")) {
parameters <- names(result_list)
values <- result_list %>% map_dbl(length) %>% as.numeric()
df_params <-
tibble(parameter = parameters, value = values)
if (df_params %>% filter(value > 1) %>% nrow() == 0) {
return(result_list)
}
double_values <-
df_params %>% filter(value > 1) %>% pull(parameter)
double_values <- double_values %>% str_to_upper()
double_values %>%
walk(function(double_value) {
double_value_slug <- str_to_lower(double_value)
value <- result_list[[double_value_slug]]
is_amount <-
double_value %in% amount_columns
is_count <-
double_value %in% count_columns
is_date <-
double_value %in% date_columns
if (is_amount) {
value <- value %>% .shorten_amounts()
result_list[[double_value_slug]] <<-
value
return(invisible())
}
if (is_date) {
value <-
value %>% .shorten_dates()
result_list[[double_value_slug]] <<-
value
return(invisible())
}
if (is_count) {
value <-
value %>% .shorten_count()
result_list[[double_value_slug]] <<-
value
return(invisible())
}
value <-
value %>% .shorten_character()
result_list[[double_value_slug]] <<-
value
return(invisible())
})
result_list
}
.create_fpds_query_params <-
function(result_list,
date_columns = c(
"VENDOR_RENEWAL_DATE",
"SOLICITATION_DATE",
"FINAL_INVOICE_PAID_DATE",
"SYS_LAST_MODIFIED_DATE",
"SIGNED_DATE",
"FUNDED_THROUGH_DATE",
"ESTIMATED_COMPLETION_DATE",
"LAST_MODIFIED_DATE",
"EFFECTIVE_DATE",
"CANCELLATION_DATE",
"LAST_DATE_TO_ORDER",
"PHYSICAL_COMPLETION_DATE",
"SOLICITATION_ISSUE_DATE",
"CREATED_DATE",
"DESTROY_DATE",
"AWARD_COMPLETION_DATE",
"LAST_MOD_DATE",
"VENDOR_REGISTRATION_DATE",
"REVEAL_DATE"
),
amount_columns = c(
"BASE_EXERCISED_OPTIONS_VALUE",
"CURRENT_CONTRACT_VALUE",
"DOLLARS_OBLIGATED",
"ANNUAL_REVENUE",
"CONTRACT_VALUE",
"FEE_RANGE_LOWER_VALUE",
"FEE_RANGE_UPPER_VALUE",
"FIXED_FEE_VALUE",
"NON_GOVERNMENT_VALUE",
"OBLIGATED_AMOUNT",
"TOTAL_CURRENT_CONTRACT_VALUE",
"TOTAL_DOLLARS_OBLIGATED",
"TOTAL_NON_GOVERNMENT_VALUE",
"TOTAL_ULTIMATE_CONTRACT_VALUE",
"ULTIMATE_CONTRACT_VALUE"
),
count_columns = c("NUMBER_OF_OFFERS_RECEIVED", "NUMBER_OF_EMPLOYEES")) {
df_params <-
result_list %>% flatten_df()
count_columns <-
count_columns %>% str_to_lower() %>% str_c(collapse = "|")
date_columns <-
date_columns %>% str_to_lower() %>% str_c(collapse = "|")
duns_columns <-
df_params %>% select(matches("duns")) %>% names()
amount_columns <-
amount_columns %>% str_to_lower() %>% str_c(collapse = "|")
date_names <-
df_params %>% select(matches(date_columns)) %>% names()
count_names <-
df_params %>% select(matches(count_columns)) %>% names()
amount_names <-
df_params %>% select(matches(amount_columns)) %>% names()
if (length(date_names) > 0) {
df_params <-
df_params %>%
mutate_at(date_names,
function(x) {
x %>% map_chr(.create_date_slug)
})
}
if (length(duns_columns) > 0) {
df_params <-
df_params %>%
mutate_at(date_names,
function(x) {
x %>% map_chr(.pad_fdps_duns)
})
}
if (length(amount_names) > 0) {
df_params <-
df_params %>%
mutate_at(amount_names,
function(x) {
x %>% map_chr(.create_value_slug)
})
}
if (length(count_names) > 0) {
df_params <-
df_params %>%
mutate_at(count_names,
function(x) {
x %>% map_chr(.create_value_slug)
})
}
df_params
}
# csv ---------------------------------------------------------------------
.read_fdps_csv <-
function(url = "https://www.fpds.gov/ezsearch/fpdsportal?s=FPDS.GOV&indexName=awardfull&templateName=CSV&q=%22Palantir+TECHNOLOGIES%22&renderer=jsp&length=677") {
options(warn = -1)
type_index <-
case_when(
url %>% str_to_lower() %>% str_detect("icd") ~ "ICD",
url %>% str_to_lower() %>% str_detect("initiative") ~ "RECOVERY",
TRUE ~ "AWARD"
)
data <- fread(url,
header = F,
verbose = F,
showProgress = F)
fpds_names <-
data %>% dplyr::slice(1) %>% t() %>% as.character()
actual_names <-
tibble(name = fpds_names) %>%
mutate(column = 1:n()) %>%
mutate(name = case_when(name %in% c("", NA) ~ str_c("X", column, sep = ""),
TRUE ~ name)) %>%
pull(name)
data <-
data %>%
dplyr::slice(2:nrow(data)) %>%
setNames(fpds_names) %>%
as_tibble(.name_repair = "unique")
select_cols <- 1:(ncol(data) - 1)
data <-
data %>%
select(select_cols)
if (type_index == "ICD") {
names(data)[[21]] <- c("nameNAICS")
names(data)[[23]] <- "nameFullPSC"
}
data <-
.munge_fpds_names(data = data)
if (data %>% hasName("amountObligation")) {
data <- data %>%
mutate(amountObligation = amountObligation %>% str_remove_all("\\$") %>% parse_number())
}
data <-
data %>%
mutate(urlCSV = url,
typeProcurement = type_index) %>%
dplyr::select(which(colMeans(is.na(.)) < 1)) %>%
dplyr::select(typeProcurement, everything())
if (data %>% hasName("codeProductService")) {
data <- data %>%
mutate(codeProductService = as.character(codeProductService))
}
data <-
data %>%
.fix_contract_name() %>%
.fix_contract_description()
data <-
data %>%
mutate_if(is.numeric, as.numeric) %>%
mutate_if(is.character,
list(function(x) {
case_when(x == "" ~ NA_character_,
TRUE ~ x)
}))
data
}
.fpds_single_csv_search <-
function(fpds_type = "award",
vendor_doing_business_as_name = NULL,
global_vendor_name = NULL,
vendor_name = NULL,
vendor_duns_number = NULL,
agency_name = NULL,
award_type = NULL,
solicitation_procedure = NULL,
contracting_office_name = NULL,
contracting_agency_name = NULL,
principal_naics_code = NULL,
award_status = NULL,
contract_type = NULL,
subcontract_plan = NULL,
contract_type_description = NULL,
type_of_contract_pricing = NULL,
contract_id = NULL,
ref_idv_contract_id = NULL,
ref_idv_agency_id = NULL,
modification_number = NULL,
contracting_agency_id = NULL,
contracting_office_id = NULL,
funding_agency_id = NULL,
funding_office_id = NULL,
funding_office_name = NULL,
agency_code = NULL,
department_id = NULL,
department_name = NULL,
last_mod_date = NULL,
last_modified_by = NULL,
award_completion_date = NULL,
created_date = NULL,
signed_date = NULL,
effective_date = NULL,
estimated_completion_date = NULL,
cancellation_date = NULL,
destroy_date = NULL,
final_invoice_paid_date = NULL,
funded_through_date = NULL,
last_modified_date = NULL,
physical_completion_date = NULL,
reveal_date = NULL,
solicitation_issue_date = NULL,
sys_last_modified_date = NULL,
vendor_registration_date = NULL,
vendor_renewal_date = NULL,
obligated_amount = NULL,
total_dollars_obligated = NULL,
current_contract_value = NULL,
total_current_contract_value = NULL,
base_exercised_options_value = NULL,
ultimate_contract_value = NULL,
total_ultimate_contract_value = NULL,
contract_fiscal_year = NULL,
created_by = NULL,
description_of_requirement = NULL,
reason_for_modification = NULL,
legislative_mandates = NULL,
local_area_set_aside = NULL,
socio_economic_indicators = NULL,
multiyear_contract = NULL,
national_interest_code = NULL,
national_interest_description = NULL,
naics_description = NULL,
product_or_service_code = NULL,
product_or_service_description = NULL,
performance_district_code = NULL,
performance_country = NULL,
performance_state_name = NULL,
parent_duns_number = NULL,
research = NULL,
vendor_address_city = NULL,
vendor_congress_district_code = NULL,
vendor_address_country_code = NULL,
vendor_address_country_name = NULL,
vendor_address_state_code = NULL,
vendor_address_state_name = NULL,
vendor_address_zip_code = NULL,
extent_competed = NULL,
number_of_offers_received = NULL,
dollars_obligated = NULL,
contract_value = NULL,
fee_range_lower_value = NULL,
fee_range_upper_value = NULL,
fixed_fee_value = NULL,
total_non_government_value = NULL,
sort_item = "Signed Date",
sort_descending = T,
return_message = T,
...) {
sort_slug <-
.sort_slug(sort_method = sort_item, sort_descending = sort_descending)
if (length(vendor_duns_number) > 0) {
vendor_duns_number <-
.pad_duns(duns = vendor_duns_number, zero_base = 9) %>% pull(slugDUNS)
}
if (length(parent_duns_number) > 0) {
parent_duns_number <-
.pad_duns(duns = parent_duns_number, zero_base = 9) %>% pull(slugDUNS)
}
query <-
.fpds_api_call_params(
contract_id = contract_id,
global_vendor_name = global_vendor_name,
contracting_agency_id = contracting_agency_id,
contracting_agency_name = contracting_agency_name,
contracting_office_id = contracting_office_id,
contract_type_description = contract_type_description,
contracting_office_name = contracting_office_name,
funding_agency_id = funding_agency_id,
funding_office_id = funding_office_id,
funding_office_name = funding_office_name,
agency_code = agency_code,
agency_name = agency_name,
department_id = department_id,
department_name = department_name,
last_mod_date = last_mod_date,
last_modified_by = last_modified_by,
award_completion_date = award_completion_date,
created_date = created_date,
signed_date = signed_date,
effective_date = effective_date,
estimated_completion_date = estimated_completion_date,
obligated_amount = obligated_amount,
ultimate_contract_value = ultimate_contract_value,
type_of_contract_pricing = type_of_contract_pricing,
award_status = award_status,
contract_type = contract_type,
description_of_requirement = description_of_requirement,
reason_for_modification = reason_for_modification,
legislative_mandates = legislative_mandates,
local_area_set_aside = local_area_set_aside,
socio_economic_indicators = socio_economic_indicators,
multiyear_contract = multiyear_contract,
national_interest_code = national_interest_code,
national_interest_description = national_interest_description,
principal_naics_code = principal_naics_code,
naics_description = naics_description,
product_or_service_code = product_or_service_code,
product_or_service_description = product_or_service_description,
performance_district_code = performance_district_code,
performance_country = performance_country,
pop_state_name = performance_state_name,
vendor_address_city = vendor_address_city,
vendor_congress_district_code = vendor_congress_district_code,
vendor_address_country_code = vendor_address_country_code,
vendor_address_country_name = vendor_address_country_name,
vendor_duns_number = vendor_duns_number,
vendor_doing_business_as_name = vendor_doing_business_as_name,
vendor_name = vendor_name,
vendor_address_state_code = vendor_address_state_code,
vendor_address_state_name = vendor_address_state_name,
vendor_address_zip_code = vendor_address_zip_code,
research = research,
cancellation_date = cancellation_date,
destroy_date = destroy_date,
final_invoice_paid_date = final_invoice_paid_date,
funded_through_date = funded_through_date,
last_modified_date = last_modified_date,
physical_completion_date = physical_completion_date,
reveal_date = reveal_date,
created_by = created_by,
solicitation_issue_date = solicitation_issue_date,
vendor_registration_date = vendor_registration_date,
base_exercised_options_value = base_exercised_options_value,
vendor_renewal_date = vendor_renewal_date,
sys_last_modified_date = vendor_renewal_date,
extent_competed = extent_competed,
return_query = T,
award_type = award_type,
solicitation_procedure = solicitation_procedure,
subcontract_plan = subcontract_plan,
number_of_offers_received = number_of_offers_received,
current_contract_value = current_contract_value ,
dollars_obligated = dollars_obligated,
contract_value = contract_value,
fee_range_lower_value = fee_range_lower_value,
fee_range_upper_value = fee_range_upper_value,
fixed_fee_value = fixed_fee_value,
total_current_contract_value = total_current_contract_value,
total_dollars_obligated = total_dollars_obligated,
total_non_government_value = total_non_government_value,
total_ultimate_contract_value = total_ultimate_contract_value,
contract_fiscal_year = contract_fiscal_year,
ref_idv_contract_id = ref_idv_contract_id,
parent_duns_number = parent_duns_number,
...
)
sort_slug <-
.sort_slug(sort_method = sort_item, sort_descending = sort_descending)
if (return_message) {
query %>% message()
}
fpds_type_slug <- fpds_type %>% str_to_lower()
if (!fpds_type_slug %in% c("award", "icd", "recovery")) {
stop("FPDS type can only be Award, ICD, or Recovery")
}
fpds_slug <-
case_when(
fpds_type_slug == "award" ~ "awardfull",
fpds_type_slug == "icd" ~ "icdindex",
TRUE ~ "initiative"
)
base_html <-
glue(
"https://www.fpds.gov/ezsearch/fpdsportal?indexName={fpds_slug}&templateName=1.5.1&s=FPDS.GOV"
) %>% as.character()
url_check <-
glue("{base_html}&q={query}") %>% as.character()
.count_results_safe <-
purrr::possibly(.count_results, 0)
results <- .count_results(url = url_check)
if (results == 0) {
"No Results" %>% message()
return(tibble())
}
max_results <- pmin(30000, results)
is_over_30k <- results > 30000
if (is_over_30k) {
glue(
"There are {comma(results, digits = 0)} results but you can only access 30,000 try detailing you search"
) %>%
message()
}
csv_slug <-
glue(
"https://www.fpds.gov/ezsearch/fpdsportal?s=fpds.gov&indexName={fpds_slug}&templateName=CSV"
)
if (is_over_30k) {
df_dates <- .generate_search_dates()
}
url <-
glue("{csv_slug}&q={query}&renderer=jsp&length={max_results}{sort_slug}") %>% as.character()
if (return_message) {
glue("Accessing {comma(results, digits = 0)} results for {URLdecode(query)}") %>% message()
}
data <-
.read_fdps_csv(url = url)
names(data) <- names(data) %>% str_replace_all("slugDUNS", "idDUNS")
duns <- names(data)[names(data) %>% str_detect("idDUNS")]
if (length(duns) > 0) {
data <- data %>%
mutate_at(duns, as.numeric)
}
data <- data %>%
munge_lite(unformat = T)
df_deps <-
data %>% distinct(nameAgencyAward)
df_deps <-
df_deps %>% separate(
nameAgencyAward,
sep = "\\,",
fill = "right",
into = c("part1", "part2"),
remove = F
) %>%
mutate_all(str_squish) %>%
mutate(nameAgencyAwardClean = case_when(is.na(part2) ~ part1,
!is.na(part2) ~ str_c(part2, part1, sep = " "))) %>%
select(nameAgencyAward, nameAgencyAwardClean)
data <-
data %>%
left_join(df_deps, by = "nameAgencyAward") %>%
select(-nameAgencyAward) %>%
rename(nameAgencyAward = nameAgencyAwardClean) %>%
select(one_of(names(data)), everything()) %>%
.add_budget_year()
date_cols <- data %>%
select(matches("date")) %>%
select_if(is.character) %>%
names()
if (length(date_cols) > 0) {
data <- data %>%
mutate_at(date_cols, mdy)
}
if (return_message) {
actions <- data %>% nrow() %>% comma(digits = 0)
contracts <-
data %>% distinct(idContractAnalysis) %>% nrow() %>% comma(digits = 0)
duns <-
data %>% distinct(idDUNS) %>% nrow() %>% comma(digits = 0)
parent_duns <-
data %>% distinct(idDUNSParent) %>% nrow() %>% comma(digits = 0)
amt <-
data$amountObligation %>% sum() %>% currency(digits = 0)
from_date <- data$dateContractSigned %>% min(na.rm = T)
to_date <- data$dateContractSigned %>% max(na.rm = T)
glue(
"\n\n{green({amt})} procured between {red({from_date})} and {red({to_date})} across {yellow(actions)} actions amongst {magenta(contracts)} contracts allocated to {cyan(duns)} distinct DUNS and {blue(parent_duns)} distinct parent DUNS\n\n"
) %>% message()
}
data %>%
mutate_if(is.numeric, as.numeric)
}
#' FPDS csv search
#'
#' Search FPDS returning CSV files.
#'
#'
#' @param fpds_type \itemize{
#' \item award
#' \item recovery
#' \item ICD
#' }
#' @param vendor_doing_business_as_name
#' @param global_vendor_name
#' @param vendor_name
#' @param parent_vendor_name
#' @param vendor_duns_number
#' @param parent_duns_number
#' @param agency_name
#' @param contracting_office_name
#' @param contracting_agency_name
#' @param principal_naics_code
#' @param award_status
#' @param contract_type
#' @param contract_type_description
#' @param type_of_contract_pricing
#' @param contract_id
#' @param ref_idv_contract_id
#' @param ref_idv_agency_id
#' @param contracting_agency_id
#' @param contracting_office_id
#' @param funding_agency_id
#' @param funding_office_id
#' @param funding_office_name
#' @param agency_code
#' @param department_id
#' @param department_name
#' @param research
#' @param last_mod_date
#' @param last_modified_by
#' @param award_completion_date
#' @param created_date
#' @param signed_date
#' @param effective_date
#' @param estimated_completion_date
#' @param obligated_amount
#' @param base_exercised_options_value
#' @param ultimate_contract_value
#' @param created_by
#' @param description_of_requirement
#' @param reason_for_modification
#' @param legislative_mandates
#' @param local_area_set_aside
#' @param socio_economic_indicators
#' @param multiyear_contract
#' @param national_interest_code
#' @param product_or_service_code
#' @param performance_district_code
#' @param performance_country
#' @param performance_state_name
#' @param vendor_address_city
#' @param vendor_congress_district_code
#' @param vendor_address_country_code
#' @param vendor_address_country_name
#' @param vendor_address_state_code
#' @param vendor_address_state_name
#' @param vendor_address_zip_code
#' @param sort_item
#' @param sort_descending
#' @param return_message
#' @param cancellation_date
#' @param destroy_date
#' @param final_invoice_paid_date
#' @param funded_through_date
#' @param last_modified_date
#' @param physical_completion_date
#' @param reveal_date
#' @param solicitation_issue_date
#' @param sys_last_modified_date
#' @param vendor_registration_date
#' @param vendor_renewal_date
#' @param award_type
#' @param solicitation_procedure
#' @param subcontract_plan
#' @param ...
#' @param current_contract_value
#' @param dollars_obligated
#' @param contract_value
#' @param fee_range_lower_value
#' @param fee_range_upper_value
#' @param fixed_fee_value
#' @param total_current_contract_value
#' @param total_dollars_obligated
#' @param total_non_government_value
#' @param total_ultimate_contract_value
#' @param contract_fiscal_year
#' @param extent_competed
#' @param number_of_offers_received
#' @param clean_entity_column if \code{TRUE} cleans entity columns
#' @param unformat
#' @param exclude_bloat
#' @param use_future
#' @param snake_names
#' @param decode_contract_ids
#'
#' @return
#' @export
#'
#' @examples
#' library(tidyverse)
#' library(govtrackR)
#'
#' ## Anything with contract signed between April 6th & 8th
#'
#' data <- fpds_csv(signed_date = "2019/04/06|2019/04/08")
#'
#'
#' ## What has the government spent on software from March 1st 2019 to April 8th 2019 vs Last Year
#'
#' df_software <- fpds_csv(principal_naics_code = 511210, signed_date = c("2019-03-01", "2019-04-08"))
#'
#' df_software %>% glimpse()
#'
#' ## Exhibit Design Architectural Services
#'
#' fpds_csv(product_or_service_code = "C1PB")
fpds_csv <-
function(fpds_type = "award",
decode_contract_ids = F,
vendor_doing_business_as_name = NA,
global_vendor_name = NA,
vendor_name = NA,
parent_vendor_name = NA,
base_exercised_options_value = NA,
current_contract_value = NA,
dollars_obligated = NA,
contract_value = NA,
fee_range_lower_value = NA,
fee_range_upper_value = NA,
fixed_fee_value = NA,
obligated_amount = NA,
total_current_contract_value = NA,
total_dollars_obligated = NA,
total_non_government_value = NA,
total_ultimate_contract_value = NA,
ultimate_contract_value = NA,
contract_fiscal_year = NA,
vendor_duns_number = NA,
parent_duns_number = NA,
agency_name = NA,
award_type = NA,
solicitation_procedure = NA,
contracting_office_name = NA,
contracting_agency_name = NA,
principal_naics_code = NA,
award_status = NA,
contract_type = NA,
contract_type_description = NA,
type_of_contract_pricing = NA,
contract_id = NA,
subcontract_plan = NA,
ref_idv_contract_id = NA,
ref_idv_agency_id = NA,
contracting_agency_id = NA,
contracting_office_id = NA,
funding_agency_id = NA,
funding_office_id = NA,
funding_office_name = NA,
agency_code = NA,
department_id = NA,
department_name = NA,
research = NA,
last_mod_date = NA,
last_modified_by = NA,
award_completion_date = NA,
created_date = NA,
signed_date = NA,
effective_date = NA,
estimated_completion_date = NA,
cancellation_date = NA,
destroy_date = NA,
final_invoice_paid_date = NA,
funded_through_date = NA,
last_modified_date = NA,
physical_completion_date = NA,
reveal_date = NA,
solicitation_issue_date = NA,
sys_last_modified_date = NA,
vendor_registration_date = NA,
vendor_renewal_date = NA,
created_by = NA,
description_of_requirement = NA,
reason_for_modification = NA,
legislative_mandates = NA,
local_area_set_aside = NA,
socio_economic_indicators = NA,
multiyear_contract = NA,
national_interest_code = NA,
product_or_service_code = NA,
performance_district_code = NA,
performance_country = NA,
performance_state_name = NA,
vendor_address_city = NA,
vendor_congress_district_code = NA,
vendor_address_country_code = NA,
vendor_address_country_name = NA,
vendor_address_state_code = NA,
vendor_address_state_name = NA,
vendor_address_zip_code = NA,
extent_competed = NA,
number_of_offers_received = NA,
sort_item = "Signed Date",
unformat = F,
exclude_bloat = T,
clean_entity_column = F,
sort_descending = T,
use_future = T,
return_message = T,
snake_names = F,
...) {
if (!is.na(contract_id) %>% sum(na.rm = T) > 0) {
contract_id <-
contract_id %>% str_remove_all("\\-")
}
if (!is.na(last_mod_date) %>% sum(na.rm = T) > 0 &
length(last_mod_date) == 2) {
last_mod_date <- .shorten_dates(dates = last_mod_date)
}
if (!is.na(award_completion_date) %>% sum(na.rm = T) > 0 &
length(award_completion_date) == 2) {
award_completion_date <-
.shorten_dates(dates = award_completion_date)
}
if (!is.na(created_date) %>% sum(na.rm = T) > 0 &
length(created_date) == 2) {
created_date <- .shorten_dates(dates = created_date)
}
if (!is.na(signed_date) %>% sum(na.rm = T) > 0 &
length(signed_date) == 2) {
signed_date <- .shorten_dates(dates = signed_date)
}
if (!is.na(effective_date) %>% sum(na.rm = T) > 0 &
length(effective_date) == 2) {
effective_date <- .shorten_dates(dates = effective_date)
}
if (!is.na(estimated_completion_date) %>% sum(na.rm = T) > 0 &
length(estimated_completion_date) == 2) {
estimated_completion_date <-
.shorten_dates(dates = estimated_completion_date)
}
if (!is.na(cancellation_date) %>% sum(na.rm = T) > 0 &
length(cancellation_date) == 2) {
cancellation_date <- .shorten_dates(dates = cancellation_date)
}
if (!is.na(destroy_date) %>% sum(na.rm = T) > 0 &
length(destroy_date) == 2) {
destroy_date <- .shorten_dates(dates = destroy_date)
}
if (!is.na(final_invoice_paid_date) %>% sum(na.rm = T) > 0 &
length(final_invoice_paid_date) == 2) {
final_invoice_paid_date <-
.shorten_dates(dates = final_invoice_paid_date)
}
if (!is.na(funded_through_date) %>% sum(na.rm = T) > 0 &
length(funded_through_date) == 2) {
funded_through_date <- .shorten_dates(dates = funded_through_date)
}
if (!is.na(last_modified_date) %>% sum(na.rm = T) > 0 &
length(last_modified_date) == 2) {
last_modified_date <- .shorten_dates(dates = last_modified_date)
}
if (!is.na(physical_completion_date) %>% sum(na.rm = T) > 0 &
length(physical_completion_date) == 2) {
physical_completion_date <-
.shorten_dates(dates = physical_completion_date)
}
if (!is.na(reveal_date) %>% sum(na.rm = T) > 0 &
length(reveal_date) == 2) {
reveal_date <- .shorten_dates(dates = reveal_date)
}
if (!is.na(solicitation_issue_date) %>% sum(na.rm = T) > 0 &
length(solicitation_issue_date) == 2) {
solicitation_issue_date <-
.shorten_dates(dates = solicitation_issue_date)
}
if (!is.na(sys_last_modified_date) %>% sum(na.rm = T) > 0 &
length(sys_last_modified_date) == 2) {
sys_last_modified_date <-
.shorten_dates(dates = sys_last_modified_date)
}
if (!is.na(vendor_registration_date) %>% sum(na.rm = T) > 0 &
length(vendor_registration_date) == 2) {
vendor_registration_date <-
.shorten_dates(dates = vendor_registration_date)
}
if (!is.na(vendor_renewal_date) %>% sum(na.rm = T) > 0 &
length(signed_date) == 2) {
vendor_renewal_date <-
.shorten_dates(dates = vendor_renewal_date)
}
if (!is.na(base_exercised_options_value) %>% sum(na.rm = T) > 0 &
length(base_exercised_options_value) == 2) {
base_exercised_options_value <-
.shorten_amounts(base_exercised_options_value)
}
if (!is.na(current_contract_value) %>% sum(na.rm = T) > 0 &
length(current_contract_value) == 2) {
current_contract_value <- .shorten_amounts(current_contract_value)
}
if (!is.na(dollars_obligated) %>% sum(na.rm = T) > 0 &
length(dollars_obligated) == 2) {
dollars_obligated <- .shorten_amounts(dollars_obligated)
}
if (!is.na(contract_value) %>% sum(na.rm = T) > 0 &
length(contract_value) == 2) {
contract_value <- .shorten_amounts(contract_value)
}
if (!is.na(fee_range_lower_value) %>% sum(na.rm = T) > 0 &
length(fee_range_lower_value) == 2) {
fee_range_lower_value <- .shorten_amounts(fee_range_lower_value)
}
if (!is.na(fee_range_upper_value) %>% sum(na.rm = T) > 0 &
length(fee_range_upper_value) == 2) {
fee_range_upper_value <- .shorten_amounts(fee_range_upper_value)
}
if (!is.na(fixed_fee_value) %>% sum(na.rm = T) > 0 &
length(fixed_fee_value) == 2) {
fixed_fee_value <- .shorten_amounts(fixed_fee_value)
}
if (!is.na(obligated_amount) %>% sum(na.rm = T) > 0 &
length(obligated_amount) == 2) {
obligated_amount <- .shorten_amounts(obligated_amount)
}
if (!is.na(total_current_contract_value) %>% sum(na.rm = T) > 0 &
length(total_current_contract_value) == 2) {
total_current_contract_value <-
.shorten_amounts(total_current_contract_value)
}
if (!is.na(total_dollars_obligated) %>% sum(na.rm = T) > 0 &
length(total_dollars_obligated) == 2) {
total_dollars_obligated <- .shorten_amounts(total_dollars_obligated)
}
if (!is.na(total_non_government_value) %>% sum(na.rm = T) > 0 &
length(total_non_government_value) == 2) {
total_non_government_value <-
.shorten_amounts(total_non_government_value)
}
if (!is.na(total_ultimate_contract_value) %>% sum(na.rm = T) > 0 &
length(total_ultimate_contract_value) == 2) {
total_ultimate_contract_value <-
.shorten_amounts(total_ultimate_contract_value)
}
if (!is.na(ultimate_contract_value) %>% sum(na.rm = T) > 0 &
length(ultimate_contract_value) == 2) {
ultimate_contract_value <-
.shorten_amounts(ultimate_contract_value)
}
df_inputs <-
expand.grid(
vendor_doing_business_as_name = vendor_doing_business_as_name,
global_vendor_name = global_vendor_name,
vendor_name = vendor_name,
award_type = award_type,
vendor_duns_number = vendor_duns_number,
agency_name = agency_name,
solicitation_procedure = solicitation_procedure,
contracting_office_name = contracting_office_name,
contracting_agency_name = contracting_agency_name,
principal_naics_code = principal_naics_code,
parent_duns_number = parent_duns_number,
parent_vendor_name = parent_vendor_name,
base_exercised_options_value = base_exercised_options_value,
current_contract_value = current_contract_value,
dollars_obligated = dollars_obligated,
contract_value = contract_value,
fee_range_lower_value = fee_range_lower_value,
fee_range_upper_value = fee_range_upper_value,
fixed_fee_value = fixed_fee_value,
obligated_amount = obligated_amount ,
total_current_contract_value = total_current_contract_value,
total_dollars_obligated = total_dollars_obligated,
total_non_government_value = total_non_government_value,
total_ultimate_contract_value = total_ultimate_contract_value,
ultimate_contract_value = ultimate_contract_value,
contract_fiscal_year = contract_fiscal_year,
award_status = award_status,
contract_type = contract_type,
contract_type_description = contract_type_description,
type_of_contract_pricing = type_of_contract_pricing,
contract_id = contract_id,
ref_idv_contract_id = ref_idv_contract_id,
ref_idv_agency_id = ref_idv_agency_id ,
contracting_agency_id = contracting_agency_id,
contracting_office_id = contracting_office_id,
funding_agency_id = funding_agency_id,
funding_office_id = funding_office_id,
funding_office_name = funding_office_name ,
agency_code = agency_code,
department_id = department_id,
department_name = department_name,
last_mod_date = last_mod_date,
last_modified_by = last_modified_by,
award_completion_date = award_completion_date,
created_date = created_date,
signed_date = signed_date ,
effective_date = effective_date,
estimated_completion_date = estimated_completion_date,
cancellation_date = cancellation_date,
destroy_date = destroy_date,
final_invoice_paid_date = final_invoice_paid_date,
funded_through_date = funded_through_date,
last_modified_date = last_modified_date,
physical_completion_date = physical_completion_date,
reveal_date = reveal_date,
solicitation_issue_date = solicitation_issue_date,
sys_last_modified_date = sys_last_modified_date,
vendor_registration_date = vendor_registration_date,
vendor_renewal_date = vendor_renewal_date,
created_by = created_by,
description_of_requirement = description_of_requirement,
reason_for_modification = reason_for_modification ,
legislative_mandates = legislative_mandates,
local_area_set_aside = local_area_set_aside ,
socio_economic_indicators = socio_economic_indicators ,
multiyear_contract = multiyear_contract,
national_interest_code = national_interest_code,
product_or_service_code = product_or_service_code,
performance_district_code = performance_district_code,
performance_country = performance_country ,
performance_state_name = performance_state_name,
vendor_address_city = vendor_address_city,
vendor_congress_district_code = vendor_congress_district_code,
vendor_address_country_code = vendor_address_country_code,
vendor_address_country_name = vendor_address_country_name,
vendor_address_state_code = vendor_address_state_code,
vendor_address_state_name = vendor_address_state_name,
vendor_address_zip_code = vendor_address_zip_code,
research = research,
subcontract_plan = subcontract_plan,
extent_competed = extent_competed,
number_of_offers_received = number_of_offers_received,
stringsAsFactors = F,
...
) %>%
as_tibble()
if (df_inputs %>% gather(item, value) %>% filter(!is.na(value)) %>% nrow() == 0) {
stop("Please enter FPDS search parameter")
}
.fpds_single_csv_search_safe <-
possibly(.fpds_single_csv_search, tibble())
if (use_future) {
all_data <-
all_data <-
1:nrow(df_inputs) %>%
future_map_dfr(function(x) {
df_row <-
df_inputs %>% dplyr::slice(x) %>% as_tibble() %>% mutate_if(is.factor, as.character)
if (is.na(df_row$vendor_doing_business_as_name)) {
vendor_doing_business_as_name_slug <- NULL
} else {
vendor_doing_business_as_name_slug <-
df_row$vendor_doing_business_as_name
}
if (is.na(df_row$global_vendor_name)) {
global_vendor_name_slug <- NULL
} else {
global_vendor_name_slug <- df_row$global_vendor_name
}
if (is.na(df_row$solicitation_procedure)) {
solicitation_procedure_slug <- NULL
} else {
solicitation_procedure_slug <- df_row$solicitation_procedure
}
if (is.na(df_row$subcontract_plan)) {
subcontract_plan_slug <- NULL
} else {
subcontract_plan_slug <- df_row$subcontract_plan
}
if (is.na(df_row$award_type)) {
award_type_slug <- NULL
} else {
award_type_slug <- df_row$award_type
}
if (is.na(df_row$research)) {
research_slug <- NULL
} else {
research_slug <- df_row$research
}
if (is.na(df_row$vendor_name)) {
vendor_name_slug <- NULL
} else {
vendor_name_slug <- df_row$vendor_name
}
if (is.na(df_row$parent_vendor_name)) {
parent_vendor_name_slug <- NULL
} else {
parent_vendor_name_slug <- df_row$parent_vendor_name
}
if (is.na(df_row$vendor_duns_number)) {
vendor_duns_number_slug <- NULL
} else {
vendor_duns_number_slug <- df_row$vendor_duns_number
}
if (is.na(df_row$parent_duns_number)) {
parent_duns_number_slug <- NULL
} else {
parent_duns_number_slug <- df_row$parent_duns_number
}
if (is.na(df_row$agency_name)) {
agency_name_slug <- NULL
} else {
agency_name_slug <- df_row$agency_name
}
if (is.na(df_row$contracting_office_name)) {
contracting_office_name_slug <- NULL
} else {
contracting_office_name_slug <- df_row$contracting_office_name
}
if (is.na(df_row$contracting_agency_name)) {
contracting_agency_name_slug <- NULL
} else {
contracting_agency_name_slug <- df_row$contracting_agency_name
}
if (is.na(df_row$principal_naics_code)) {
principal_naics_code_slug <- NULL
} else {
principal_naics_code_slug <- df_row$principal_naics_code
}
if (is.na(df_row$award_status)) {
award_status_slug <- NULL
} else {
award_status_slug <- df_row$award_status
}
if (is.na(df_row$contract_type)) {
contract_type_slug <- NULL
} else {
contract_type_slug <- df_row$contract_type
}
if (is.na(df_row$contract_type_description)) {
contract_type_description_slug <- NULL
} else {
contract_type_description_slug <- df_row$contract_type_description
}
if (is.na(df_row$type_of_contract_pricing)) {
type_of_contract_pricing_slug <- NULL
} else {
type_of_contract_pricing_slug <- df_row$type_of_contract_pricing
}
if (is.na(df_row$contract_id)) {
contract_id_slug <- NULL
} else {
contract_id_slug <- df_row$contract_id
}
if (is.na(df_row$ref_idv_contract_id)) {
ref_idv_contract_id_slug <- NULL
} else {
ref_idv_contract_id_slug <- df_row$ref_idv_contract_id
}
if (is.na(df_row$ref_idv_agency_id)) {
ref_idv_agency_id_slug <- NULL
} else {
ref_idv_agency_id_slug <- df_row$ref_idv_agency_id
}
if (is.na(df_row$contracting_agency_id)) {
contracting_agency_id_slug <- NULL
} else {
contracting_agency_id_slug <- df_row$contracting_agency_id
}
if (is.na(df_row$contracting_office_id)) {
contracting_office_id_slug <- NULL
} else {
contracting_office_id_slug <- df_row$contracting_office_id
}
if (is.na(df_row$funding_agency_id)) {
funding_agency_id_slug <- NULL
} else {
funding_agency_id_slug <- df_row$funding_agency_id
}
if (is.na(df_row$funding_office_id)) {
funding_office_id_slug <- NULL
} else {
funding_office_id_slug <- df_row$funding_office_id
}
if (is.na(df_row$funding_office_name)) {
funding_office_name_slug <- NULL
} else {
funding_office_name_slug <- df_row$funding_office_name
}
if (is.na(df_row$agency_code)) {
agency_code_slug <- NULL
} else {
agency_code_slug <- df_row$agency_code
}
if (is.na(df_row$department_id)) {
department_id_slug <- NULL
} else {
department_id_slug <- df_row$department_id
}
if (is.na(df_row$department_name)) {
department_name_slug <- NULL
} else {
department_name_slug <- df_row$department_name
}
if (is.na(df_row$last_mod_date)) {
last_mod_date_slug <- NULL
} else {
last_mod_date_slug <- df_row$last_mod_date
}
if (is.na(df_row$last_modified_by)) {
last_modified_by_slug <- NULL
} else {
last_modified_by_slug <- df_row$last_modified_by
}
if (is.na(df_row$award_completion_date)) {
award_completion_date_slug <- NULL
} else {
award_completion_date_slug <- df_row$award_completion_date
}
if (is.na(df_row$created_date)) {
created_date_slug <- NULL
} else {
created_date_slug <- df_row$created_date
}
if (is.na(df_row$signed_date)) {
signed_date_slug <- NULL
} else {
signed_date_slug <- df_row$signed_date
}
if (is.na(df_row$effective_date)) {
effective_date_slug <- NULL
} else {
effective_date_slug <- df_row$effective_date
}
if (is.na(df_row$estimated_completion_date)) {
estimated_completion_date_slug <- NULL
} else {
estimated_completion_date_slug <- df_row$estimated_completion_date
}
if (is.na(df_row$cancellation_date)) {
cancellation_date_slug <- NULL
} else {
cancellation_date_slug <- df_row$cancellation_date
}
if (is.na(df_row$destroy_date)) {
destroy_date_slug <- NULL
} else {
destroy_date_slug <- df_row$destroy_date
}
if (is.na(df_row$final_invoice_paid_date)) {
final_invoice_paid_date_slug <- NULL
} else {
final_invoice_paid_date_slug <- df_row$final_invoice_paid_date
}
if (is.na(df_row$funded_through_date)) {
funded_through_date_slug <- NULL
} else {
funded_through_date_slug <- df_row$funded_through_date
}
if (is.na(df_row$last_modified_date)) {
last_modified_date_slug <- NULL
} else {
last_modified_date_slug <- df_row$last_modified_date
}
if (is.na(df_row$physical_completion_date)) {
physical_completion_date_slug <- NULL
} else {
physical_completion_date_slug <- df_row$physical_completion_date
}
if (is.na(df_row$reveal_date)) {
reveal_date_slug <- NULL
} else {
reveal_date_slug <- df_row$reveal_date
}
if (is.na(df_row$solicitation_issue_date)) {
solicitation_issue_date_slug <- NULL
} else {
solicitation_issue_date_slug <- df_row$solicitation_issue_date
}
if (is.na(df_row$sys_last_modified_date)) {
sys_last_modified_date_slug <- NULL
} else {
sys_last_modified_date_slug <- df_row$sys_last_modified_date
}
if (is.na(df_row$vendor_registration_date)) {
vendor_registration_date_slug <- NULL
} else {
vendor_registration_date_slug <- df_row$vendor_registration_date
}
if (is.na(df_row$vendor_renewal_date)) {
vendor_renewal_date_slug <- NULL
} else {
vendor_renewal_date_slug <-
df_row$vendor_revendor_renewal_dategistration_date
}
if (is.na(df_row$obligated_amount)) {
obligated_amount_slug <- NULL
} else {
obligated_amount_slug <- df_row$obligated_amount
}
if (is.na(df_row$base_exercised_options_value)) {
base_exercised_options_value_slug <- NULL
} else {
base_exercised_options_value_slug <-
df_row$base_exercised_options_value
}
if (is.na(df_row$ultimate_contract_value)) {
ultimate_contract_value_slug <- NULL
} else {
ultimate_contract_value_slug <- df_row$ultimate_contract_value
}
if (is.na(df_row$created_by)) {
created_by_slug <- NULL
} else {
created_by_slug <- df_row$created_by
}
if (is.na(df_row$description_of_requirement)) {
description_of_requirement_slug <- NULL
} else {
description_of_requirement_slug <- df_row$description_of_requirement
}
if (is.na(df_row$legislative_mandates)) {
legislative_mandates_slug <- NULL
} else {
legislative_mandates_slug <- df_row$legislative_mandates
}
if (is.na(df_row$local_area_set_aside)) {
local_area_set_aside_slug <- NULL
} else {
local_area_set_aside_slug <- df_row$local_area_set_aside
}
if (is.na(df_row$socio_economic_indicators)) {
socio_economic_indicators_slug <- NULL
} else {
socio_economic_indicators_slug <- df_row$socio_economic_indicators
}
if (is.na(df_row$multiyear_contract)) {
multiyear_contract_slug <- NULL
} else {
multiyear_contract_slug <- df_row$multiyear_contract
}
if (is.na(df_row$national_interest_code)) {
national_interest_code_slug <- NULL
} else {
national_interest_code_slug <- df_row$national_interest_code
}
if (is.na(df_row$product_or_service_code)) {
product_or_service_code_slug <- NULL
} else {
product_or_service_code_slug <- df_row$product_or_service_code
}
if (is.na(df_row$performance_district_code)) {
performance_district_code_slug <- NULL
} else {
performance_district_code_slug <- df_row$performance_district_code
}
if (is.na(df_row$performance_country)) {
performance_country_slug <- NULL
} else {
performance_country_slug <- df_row$performance_country
}
if (is.na(df_row$performance_state_name)) {
performance_state_name_slug <- NULL
} else {
performance_state_name_slug <- df_row$performance_state_name
}
if (is.na(df_row$vendor_address_city)) {
vendor_address_city_slug <- NULL
} else {
vendor_address_city_slug <- df_row$vendor_address_city
}
if (is.na(df_row$vendor_congress_district_code)) {
vendor_congress_district_code_slug <- NULL
} else {
vendor_congress_district_code_slug <-
df_row$vendor_congress_district_code
}
if (is.na(df_row$vendor_address_country_code)) {
vendor_address_country_code_slug <- NULL
} else {
vendor_address_country_code_slug <-
df_row$vendor_address_country_code
}
if (is.na(vendor_address_country_name)) {
vendor_address_country_name_slug <- NULL
} else {
vendor_address_country_name_slug <-
df_row$vendor_address_country_name
}
if (is.na(df_row$vendor_address_state_code)) {
vendor_address_state_code_slug <- NULL
} else {
vendor_address_state_code_slug <- df_row$vendor_address_state_code
}
if (is.na(df_row$vendor_address_state_name)) {
vendor_address_state_name_slug <- NULL
} else {
vendor_address_state_name_slug <- df_row$vendor_address_state_name
}
if (is.na(df_row$vendor_address_zip_code)) {
vendor_address_zip_code_slug <- NULL
} else {
vendor_address_zip_code_slug <- df_row$vendor_address_zip_code
}
if (is.na(df_row$reason_for_modification)) {
reason_for_modification_slug <- NULL
} else {
reason_for_modification_slug <- df_row$reason_for_modification
}
if (is.na(current_contract_value)) {
current_contract_value_slug <- NULL
} else {
current_contract_value_slug <- df_row$current_contract_value
}
if (is.na(dollars_obligated)) {
dollars_obligated_slug <- NULL
} else {
dollars_obligated_slug <- df_row$dollars_obligated
}
if (is.na(contract_value)) {
contract_value_slug <- NULL
} else {
contract_value_slug <- df_row$contract_value
}
if (is.na(fee_range_lower_value)) {
fee_range_lower_value_slug <- NULL
} else {
fee_range_lower_value_slug <- df_row$fee_range_lower_value
}
if (is.na(fee_range_upper_value)) {
fee_range_upper_value_slug <- NULL
} else {
}
if (is.na(fixed_fee_value)) {
fixed_fee_value_slug <- NULL
} else {
fixed_fee_value_slug <- df_row$fixed_fee_value
}
if (is.na(total_current_contract_value)) {
total_current_contract_value_slug <- NULL
} else {
total_current_contract_value_slug <-
df_row$total_current_contract_value
}
if (is.na(total_dollars_obligated)) {
total_dollars_obligated_slug <- NULL
} else {
total_dollars_obligated_slug <- df_row$total_dollars_obligated
}
if (is.na(total_non_government_value)) {
total_non_government_value_slug <- NULL
} else {
total_non_government_value_slug <- df_row$total_non_government_value
}
if (is.na(total_ultimate_contract_value)) {
total_ultimate_contract_value_slug <- NULL
} else {
total_ultimate_contract_value_slug <-
df_row$total_ultimate_contract_value
}
if (is.na(contract_fiscal_year)) {
contract_fiscal_year_slug <- NULL
} else {
contract_fiscal_year_slug <- df_row$contract_fiscal_year
}
if (is.na(extent_competed)) {
extent_competed_slug <- NULL
} else {
extent_competed_slug <- df_row$extent_competed
}
if (is.na(number_of_offers_received)) {
number_of_offers_received_slug <- NULL
} else {
number_of_offers_received_slug <- df_row$number_of_offers_received
}
d <-
.fpds_single_csv_search_safe(
fpds_type = fpds_type,
vendor_doing_business_as_name = vendor_doing_business_as_name_slug,
award_type = award_type_slug,
global_vendor_name = global_vendor_name_slug,
vendor_name = vendor_name_slug,
parent_vendor_name = parent_vendor_name_slug,
vendor_duns_number = vendor_duns_number_slug,
parent_duns_number = parent_duns_number_slug,
agency_name = agency_name_slug,
contracting_office_name = contracting_office_name_slug,
contracting_agency_name = contracting_agency_name_slug,
principal_naics_code = principal_naics_code_slug,
award_status = award_status_slug,
contract_type = contract_type_slug,
contract_type_description = contract_type_description_slug,
type_of_contract_pricing = type_of_contract_pricing_slug,
contract_id = contract_id_slug,
ref_idv_contract_id = ref_idv_contract_id_slug,
ref_idv_agency_id = ref_idv_agency_id_slug,
contracting_agency_id = contracting_agency_id_slug,
contracting_office_id = contracting_office_id_slug,
funding_agency_id = funding_agency_id_slug,
funding_office_id = funding_office_id_slug,
funding_office_name = funding_office_name_slug,
agency_code = agency_code_slug,
department_id = department_id_slug,
department_name = department_name_slug,
research = research_slug,
last_mod_date = last_mod_date_slug,
last_modified_by = last_modified_by_slug,
award_completion_date = award_completion_date_slug,
created_date = created_date_slug,
signed_date = signed_date_slug,
effective_date = effective_date_slug,
estimated_completion_date = estimated_completion_date_slug,
obligated_amount = obligated_amount_slug,
base_exercised_options_value = base_exercised_options_value_slug,
ultimate_contract_value = ultimate_contract_value_slug,
created_by = created_by_slug,
description_of_requirement = description_of_requirement_slug,
reason_for_modification = reason_for_modification_slug,
legislative_mandates = legislative_mandates_slug,
local_area_set_aside = local_area_set_aside_slug,
socio_economic_indicators = socio_economic_indicators_slug,
multiyear_contract = multiyear_contract_slug,
national_interest_code = national_interest_code_slug,
product_or_service_code = product_or_service_code_slug,
performance_district_code = performance_district_code_slug,
performance_country = performance_country_slug,
performance_state_name = performance_state_name_slug,
vendor_address_city = vendor_address_city_slug,
vendor_congress_district_code = vendor_congress_district_code_slug,
vendor_address_country_code = vendor_address_country_code_slug,
vendor_address_country_name = vendor_address_country_name_slug,
vendor_address_state_code = vendor_address_state_code_slug,
vendor_address_state_name = vendor_address_state_name_slug,
vendor_address_zip_code = vendor_address_zip_code_slug,
sort_item = sort_item,
sort_descending = sort_descending,
return_message = return_message,
cancellation_date = cancellation_date_slug,
destroy_date = destroy_date_slug,
final_invoice_paid_date = final_invoice_paid_date_slug,
funded_through_date = funded_through_date_slug,
last_modified_date = last_modified_date_slug,
physical_completion_date = physical_completion_date_slug,
reveal_date = reveal_date_slug,
solicitation_issue_date = solicitation_issue_date_slug,
sys_last_modified_date = sys_last_modified_date_slug,
vendor_registration_date = vendor_registration_date_slug,
vendor_renewal_date = vendor_renewal_date_slug,
solicitation_procedure = solicitation_procedure_slug,
subcontract_plan = subcontract_plan_slug,
total_dollars_obligated = total_dollars_obligated_slug,
current_contract_value = current_contract_value_slug,
total_current_contract_value = total_current_contract_value_slug,
total_ultimate_contract_value = total_ultimate_contract_value_slug,
contract_fiscal_year = contract_fiscal_year_slug,
...
)
if (d %>% hasName("codeModification")) {
d <- d %>%
mutate(codeModification = as.character(codeModification))
}
if (d %>% tibble::has_name("idAgencyAward")) {
d <- d %>%
mutate(idAgencyAward = as.character(idAgencyAward))
}
if (!d %>% tibble::has_name("zipcodeVendor")) {
d <- d %>%
mutate(zipcodeVendor = NA_character_)
}
if (d %>% hasName("codeProductService")) {
d <- d %>%
mutate(codeProductService = as.character(codeProductService))
}
d <- d %>%
mutate(zipcodeVendor = as.character(zipcodeVendor))
d
})
} else {
all_data <-
1:nrow(df_inputs) %>%
map_dfr(function(x) {
df_row <-
df_inputs %>% dplyr::slice(x) %>% as_tibble() %>% mutate_if(is.factor, as.character)
if (is.na(df_row$vendor_doing_business_as_name)) {
vendor_doing_business_as_name_slug <- NULL
} else {
vendor_doing_business_as_name_slug <-
df_row$vendor_doing_business_as_name
}
if (is.na(df_row$global_vendor_name)) {
global_vendor_name_slug <- NULL
} else {
global_vendor_name_slug <- df_row$global_vendor_name
}
if (is.na(df_row$solicitation_procedure)) {
solicitation_procedure_slug <- NULL
} else {
solicitation_procedure_slug <- df_row$solicitation_procedure
}
if (is.na(df_row$subcontract_plan)) {
subcontract_plan_slug <- NULL
} else {
subcontract_plan_slug <- df_row$subcontract_plan
}
if (is.na(df_row$award_type)) {
award_type_slug <- NULL
} else {
award_type_slug <- df_row$award_type
}
if (is.na(df_row$research)) {
research_slug <- NULL
} else {
research_slug <- df_row$research
}
if (is.na(df_row$vendor_name)) {
vendor_name_slug <- NULL
} else {
vendor_name_slug <- df_row$vendor_name
}
if (is.na(df_row$parent_vendor_name)) {
parent_vendor_name_slug <- NULL
} else {
parent_vendor_name_slug <- df_row$parent_vendor_name
}
if (is.na(df_row$vendor_duns_number)) {
vendor_duns_number_slug <- NULL
} else {
vendor_duns_number_slug <- df_row$vendor_duns_number
}
if (is.na(df_row$parent_duns_number)) {
parent_duns_number_slug <- NULL
} else {
parent_duns_number_slug <- df_row$parent_duns_number
}
if (is.na(df_row$agency_name)) {
agency_name_slug <- NULL
} else {
agency_name_slug <- df_row$agency_name
}
if (is.na(df_row$contracting_office_name)) {
contracting_office_name_slug <- NULL
} else {
contracting_office_name_slug <- df_row$contracting_office_name
}
if (is.na(df_row$contracting_agency_name)) {
contracting_agency_name_slug <- NULL
} else {
contracting_agency_name_slug <- df_row$contracting_agency_name
}
if (is.na(df_row$principal_naics_code)) {
principal_naics_code_slug <- NULL
} else {
principal_naics_code_slug <- df_row$principal_naics_code
}
if (is.na(df_row$award_status)) {
award_status_slug <- NULL
} else {
award_status_slug <- df_row$award_status
}
if (is.na(df_row$contract_type)) {
contract_type_slug <- NULL
} else {
contract_type_slug <- df_row$contract_type
}
if (is.na(df_row$contract_type_description)) {
contract_type_description_slug <- NULL
} else {
contract_type_description_slug <- df_row$contract_type_description
}
if (is.na(df_row$type_of_contract_pricing)) {
type_of_contract_pricing_slug <- NULL
} else {
type_of_contract_pricing_slug <- df_row$type_of_contract_pricing
}
if (is.na(df_row$contract_id)) {
contract_id_slug <- NULL
} else {
contract_id_slug <- df_row$contract_id
}
if (is.na(df_row$ref_idv_contract_id)) {
ref_idv_contract_id_slug <- NULL
} else {
ref_idv_contract_id_slug <- df_row$ref_idv_contract_id
}
if (is.na(df_row$ref_idv_agency_id)) {
ref_idv_agency_id_slug <- NULL
} else {
ref_idv_agency_id_slug <- df_row$ref_idv_agency_id
}
if (is.na(df_row$contracting_agency_id)) {
contracting_agency_id_slug <- NULL
} else {
contracting_agency_id_slug <- df_row$contracting_agency_id
}
if (is.na(df_row$contracting_office_id)) {
contracting_office_id_slug <- NULL
} else {
contracting_office_id_slug <- df_row$contracting_office_id
}
if (is.na(df_row$funding_agency_id)) {
funding_agency_id_slug <- NULL
} else {
funding_agency_id_slug <- df_row$funding_agency_id
}
if (is.na(df_row$funding_office_id)) {
funding_office_id_slug <- NULL
} else {
funding_office_id_slug <- df_row$funding_office_id
}
if (is.na(df_row$funding_office_name)) {
funding_office_name_slug <- NULL
} else {
funding_office_name_slug <- df_row$funding_office_name
}
if (is.na(df_row$agency_code)) {
agency_code_slug <- NULL
} else {
agency_code_slug <- df_row$agency_code
}
if (is.na(df_row$department_id)) {
department_id_slug <- NULL
} else {
department_id_slug <- df_row$department_id
}
if (is.na(df_row$department_name)) {
department_name_slug <- NULL
} else {
department_name_slug <- df_row$department_name
}
if (is.na(df_row$last_mod_date)) {
last_mod_date_slug <- NULL
} else {
last_mod_date_slug <- df_row$last_mod_date
}
if (is.na(df_row$last_modified_by)) {
last_modified_by_slug <- NULL
} else {
last_modified_by_slug <- df_row$last_modified_by
}
if (is.na(df_row$award_completion_date)) {
award_completion_date_slug <- NULL
} else {
award_completion_date_slug <- df_row$award_completion_date
}
if (is.na(df_row$created_date)) {
created_date_slug <- NULL
} else {
created_date_slug <- df_row$created_date
}
if (is.na(df_row$signed_date)) {
signed_date_slug <- NULL
} else {
signed_date_slug <- df_row$signed_date
}
if (is.na(df_row$effective_date)) {
effective_date_slug <- NULL
} else {
effective_date_slug <- df_row$effective_date
}
if (is.na(df_row$estimated_completion_date)) {
estimated_completion_date_slug <- NULL
} else {
estimated_completion_date_slug <- df_row$estimated_completion_date
}
if (is.na(df_row$cancellation_date)) {
cancellation_date_slug <- NULL
} else {
cancellation_date_slug <- df_row$cancellation_date
}
if (is.na(df_row$destroy_date)) {
destroy_date_slug <- NULL
} else {
destroy_date_slug <- df_row$destroy_date
}
if (is.na(df_row$final_invoice_paid_date)) {
final_invoice_paid_date_slug <- NULL
} else {
final_invoice_paid_date_slug <- df_row$final_invoice_paid_date
}
if (is.na(df_row$funded_through_date)) {
funded_through_date_slug <- NULL
} else {
funded_through_date_slug <- df_row$funded_through_date
}
if (is.na(df_row$last_modified_date)) {
last_modified_date_slug <- NULL
} else {
last_modified_date_slug <- df_row$last_modified_date
}
if (is.na(df_row$physical_completion_date)) {
physical_completion_date_slug <- NULL
} else {
physical_completion_date_slug <- df_row$physical_completion_date
}
if (is.na(df_row$reveal_date)) {
reveal_date_slug <- NULL
} else {
reveal_date_slug <- df_row$reveal_date
}
if (is.na(df_row$solicitation_issue_date)) {
solicitation_issue_date_slug <- NULL
} else {
solicitation_issue_date_slug <- df_row$solicitation_issue_date
}
if (is.na(df_row$sys_last_modified_date)) {
sys_last_modified_date_slug <- NULL
} else {
sys_last_modified_date_slug <- df_row$sys_last_modified_date
}
if (is.na(df_row$vendor_registration_date)) {
vendor_registration_date_slug <- NULL
} else {
vendor_registration_date_slug <- df_row$vendor_registration_date
}
if (is.na(df_row$vendor_renewal_date)) {
vendor_renewal_date_slug <- NULL
} else {
vendor_renewal_date_slug <-
df_row$vendor_revendor_renewal_dategistration_date
}
if (is.na(df_row$obligated_amount)) {
obligated_amount_slug <- NULL
} else {
obligated_amount_slug <- df_row$obligated_amount
}
if (is.na(df_row$base_exercised_options_value)) {
base_exercised_options_value_slug <- NULL
} else {
base_exercised_options_value_slug <-
df_row$base_exercised_options_value
}
if (is.na(df_row$ultimate_contract_value)) {
ultimate_contract_value_slug <- NULL
} else {
ultimate_contract_value_slug <- df_row$ultimate_contract_value
}
if (is.na(df_row$created_by)) {
created_by_slug <- NULL
} else {
created_by_slug <- df_row$created_by
}
if (is.na(df_row$description_of_requirement)) {
description_of_requirement_slug <- NULL
} else {
description_of_requirement_slug <- df_row$description_of_requirement
}
if (is.na(df_row$legislative_mandates)) {
legislative_mandates_slug <- NULL
} else {
legislative_mandates_slug <- df_row$legislative_mandates
}
if (is.na(df_row$local_area_set_aside)) {
local_area_set_aside_slug <- NULL
} else {
local_area_set_aside_slug <- df_row$local_area_set_aside
}
if (is.na(df_row$socio_economic_indicators)) {
socio_economic_indicators_slug <- NULL
} else {
socio_economic_indicators_slug <- df_row$socio_economic_indicators
}
if (is.na(df_row$multiyear_contract)) {
multiyear_contract_slug <- NULL
} else {
multiyear_contract_slug <- df_row$multiyear_contract
}
if (is.na(df_row$national_interest_code)) {
national_interest_code_slug <- NULL
} else {
national_interest_code_slug <- df_row$national_interest_code
}
if (is.na(df_row$product_or_service_code)) {
product_or_service_code_slug <- NULL
} else {
product_or_service_code_slug <- df_row$product_or_service_code
}
if (is.na(df_row$performance_district_code)) {
performance_district_code_slug <- NULL
} else {
performance_district_code_slug <- df_row$performance_district_code
}
if (is.na(df_row$performance_country)) {
performance_country_slug <- NULL
} else {
performance_country_slug <- df_row$performance_country
}
if (is.na(df_row$performance_state_name)) {
performance_state_name_slug <- NULL
} else {
performance_state_name_slug <- df_row$performance_state_name
}
if (is.na(df_row$vendor_address_city)) {
vendor_address_city_slug <- NULL
} else {
vendor_address_city_slug <- df_row$vendor_address_city
}
if (is.na(df_row$vendor_congress_district_code)) {
vendor_congress_district_code_slug <- NULL
} else {
vendor_congress_district_code_slug <-
df_row$vendor_congress_district_code
}
if (is.na(df_row$vendor_address_country_code)) {
vendor_address_country_code_slug <- NULL
} else {
vendor_address_country_code_slug <-
df_row$vendor_address_country_code
}
if (is.na(vendor_address_country_name)) {
vendor_address_country_name_slug <- NULL
} else {
vendor_address_country_name_slug <-
df_row$vendor_address_country_name
}
if (is.na(df_row$vendor_address_state_code)) {
vendor_address_state_code_slug <- NULL
} else {
vendor_address_state_code_slug <- df_row$vendor_address_state_code
}
if (is.na(df_row$vendor_address_state_name)) {
vendor_address_state_name_slug <- NULL
} else {
vendor_address_state_name_slug <- df_row$vendor_address_state_name
}
if (is.na(df_row$vendor_address_zip_code)) {
vendor_address_zip_code_slug <- NULL
} else {
vendor_address_zip_code_slug <- df_row$vendor_address_zip_code
}
if (is.na(df_row$reason_for_modification)) {
reason_for_modification_slug <- NULL
} else {
reason_for_modification_slug <- df_row$reason_for_modification
}
if (is.na(current_contract_value)) {
current_contract_value_slug <- NULL
} else {
current_contract_value_slug <- df_row$current_contract_value
}
if (is.na(dollars_obligated)) {
dollars_obligated_slug <- NULL
} else {
dollars_obligated_slug <- df_row$dollars_obligated
}
if (is.na(contract_value)) {
contract_value_slug <- NULL
} else {
contract_value_slug <- df_row$contract_value
}
if (is.na(fee_range_lower_value)) {
fee_range_lower_value_slug <- NULL
} else {
fee_range_lower_value_slug <- df_row$fee_range_lower_value
}
if (is.na(fee_range_upper_value)) {
fee_range_upper_value_slug <- NULL
} else {
}
if (is.na(fixed_fee_value)) {
fixed_fee_value_slug <- NULL
} else {
fixed_fee_value_slug <- df_row$fixed_fee_value
}
if (is.na(total_current_contract_value)) {
total_current_contract_value_slug <- NULL
} else {
total_current_contract_value_slug <-
df_row$total_current_contract_value
}
if (is.na(total_dollars_obligated)) {
total_dollars_obligated_slug <- NULL
} else {
total_dollars_obligated_slug <- df_row$total_dollars_obligated
}
if (is.na(total_non_government_value)) {
total_non_government_value_slug <- NULL
} else {
total_non_government_value_slug <- df_row$total_non_government_value
}
if (is.na(total_ultimate_contract_value)) {
total_ultimate_contract_value_slug <- NULL
} else {
total_ultimate_contract_value_slug <-
df_row$total_ultimate_contract_value
}
if (is.na(contract_fiscal_year)) {
contract_fiscal_year_slug <- NULL
} else {
contract_fiscal_year_slug <- df_row$contract_fiscal_year
}
if (is.na(extent_competed)) {
extent_competed_slug <- NULL
} else {
extent_competed_slug <- df_row$extent_competed
}
if (is.na(number_of_offers_received)) {
number_of_offers_received_slug <- NULL
} else {
number_of_offers_received_slug <- df_row$number_of_offers_received
}
d <-
.fpds_single_csv_search_safe(
fpds_type = fpds_type,
vendor_doing_business_as_name = vendor_doing_business_as_name_slug,
award_type = award_type_slug,
global_vendor_name = global_vendor_name_slug,
vendor_name = vendor_name_slug,
parent_vendor_name = parent_vendor_name_slug,
vendor_duns_number = vendor_duns_number_slug,
parent_duns_number = parent_duns_number_slug,
agency_name = agency_name_slug,
contracting_office_name = contracting_office_name_slug,
contracting_agency_name = contracting_agency_name_slug,
principal_naics_code = principal_naics_code_slug,
award_status = award_status_slug,
contract_type = contract_type_slug,
contract_type_description = contract_type_description_slug,
type_of_contract_pricing = type_of_contract_pricing_slug,
contract_id = contract_id_slug,
ref_idv_contract_id = ref_idv_contract_id_slug,
ref_idv_agency_id = ref_idv_agency_id_slug,
contracting_agency_id = contracting_agency_id_slug,
contracting_office_id = contracting_office_id_slug,
funding_agency_id = funding_agency_id_slug,
funding_office_id = funding_office_id_slug,
funding_office_name = funding_office_name_slug,
agency_code = agency_code_slug,
department_id = department_id_slug,
department_name = department_name_slug,
research = research_slug,
last_mod_date = last_mod_date_slug,
last_modified_by = last_modified_by_slug,
award_completion_date = award_completion_date_slug,
created_date = created_date_slug,
signed_date = signed_date_slug,
effective_date = effective_date_slug,
estimated_completion_date = estimated_completion_date_slug,
obligated_amount = obligated_amount_slug,
base_exercised_options_value = base_exercised_options_value_slug,
ultimate_contract_value = ultimate_contract_value_slug,
created_by = created_by_slug,
description_of_requirement = description_of_requirement_slug,
reason_for_modification = reason_for_modification_slug,
legislative_mandates = legislative_mandates_slug,
local_area_set_aside = local_area_set_aside_slug,
socio_economic_indicators = socio_economic_indicators_slug,
multiyear_contract = multiyear_contract_slug,
national_interest_code = national_interest_code_slug,
product_or_service_code = product_or_service_code_slug,
performance_district_code = performance_district_code_slug,
performance_country = performance_country_slug,
performance_state_name = performance_state_name_slug,
vendor_address_city = vendor_address_city_slug,
vendor_congress_district_code = vendor_congress_district_code_slug,
vendor_address_country_code = vendor_address_country_code_slug,
vendor_address_country_name = vendor_address_country_name_slug,
vendor_address_state_code = vendor_address_state_code_slug,
vendor_address_state_name = vendor_address_state_name_slug,
vendor_address_zip_code = vendor_address_zip_code_slug,
sort_item = sort_item,
sort_descending = sort_descending,
return_message = return_message,
cancellation_date = cancellation_date_slug,
destroy_date = destroy_date_slug,
final_invoice_paid_date = final_invoice_paid_date_slug,
funded_through_date = funded_through_date_slug,
last_modified_date = last_modified_date_slug,
physical_completion_date = physical_completion_date_slug,
reveal_date = reveal_date_slug,
solicitation_issue_date = solicitation_issue_date_slug,
sys_last_modified_date = sys_last_modified_date_slug,
vendor_registration_date = vendor_registration_date_slug,
vendor_renewal_date = vendor_renewal_date_slug,
solicitation_procedure = solicitation_procedure_slug,
subcontract_plan = subcontract_plan_slug,
total_dollars_obligated = total_dollars_obligated_slug,
current_contract_value = current_contract_value_slug,
total_current_contract_value = total_current_contract_value_slug,
total_ultimate_contract_value = total_ultimate_contract_value_slug,
contract_fiscal_year = contract_fiscal_year_slug,
...
)
if (d %>% hasName("codeModification")) {
d <- d %>%
mutate(codeModification = as.character(codeModification))
}
if (d %>% tibble::has_name("idAgencyAward")) {
d <- d %>%
mutate(idAgencyAward = as.character(idAgencyAward))
}
if (!d %>% tibble::has_name("zipcodeVendor")) {
d <- d %>%
mutate(zipcodeVendor = NA_character_)
}
if (d %>% hasName("codeProductService")) {
d <- d %>%
mutate(codeProductService = as.character(codeProductService))
}
d <- d %>%
mutate(zipcodeVendor = as.character(zipcodeVendor))
d
})
}
if (nrow(all_data) == 0) {
message("No results")
return(invisible())
}
all_data <-
all_data %>%
.munge_data(clean_address = F, exclude_bloat = exclude_bloat)
df_psc <- dictionary_psc_active()
all_data <-
all_data %>%
left_join(
df_psc %>% select(typePSC,
codeProductService,
nameProductService) %>% mutate_all(str_to_upper)
,
by = "codeProductService"
) %>%
distinct() %>%
select(-one_of("descriptionProductService"))
all_data <-
all_data %>%
.remove_na()
if (all_data %>% hasName("codeProductService")) {
all_data <- all_data %>%
mutate(
idSolicitationGroup =
case_when(
is.na(codeProductService) ~ NA_character_,
codeProductService %>% substr(1, 1) %>% str_detect("[A-Z]") ~ codeProductService %>% substr(1, 1),
TRUE ~ codeProductService %>% substr(1, 2)
)
)
all_data <- all_data %>%
left_join(df_psc %>% distinct(idSolicitationGroup, nameSolicitationGroup),
by = "idSolicitationGroup")
}
if (all_data %>% hasName("nameVendorParent") &&
all_data %>% hasName("nameVendorParent")) {
all_data <-
all_data %>%
mutate(
nameVendor = case_when(
is.na(nameVendor) & !is.na(nameVendorParent) ~ nameVendorParent,
is.na(nameVendor) &
is.na(nameVendor) ~ "UNDISCLOSED VENDOR",
TRUE ~ nameVendor
),
nameVendorParent = case_when(
is.na(nameVendorParent) & !is.na(nameVendor) ~ nameVendorParent,
is.na(nameVendor) &
is.na(nameVendor) ~ "UNDISCLOSED VENDOR PARENT",
TRUE ~ nameVendorParent
)
)
}
if (all_data %>% hasName("dateContractSigned")) {
all_data <-
all_data %>%
mutate(dateObligation = dateContractSigned)
}
if (all_data %>% hasName("idDUNS") &
all_data %>% hasName("idDUNSParent")) {
all_data <-
all_data %>%
mutate_at(c("idDUNS", "idDUNSParent"),
list(function(x) {
ifelse(x == 0, NA_integer_, x)
})) %>%
mutate(hasParent = idDUNSParent != idDUNS)
}
if (all_data %>% hasName("idContractIDV")) {
all_data <- all_data %>%
mutate(isIDV = !is.na(idContractIDV))
}
if (clean_entity_column) {
all_data <-
.clean_org_usa(data = all_data,
col = "nameAgencyAward",
clean_entity_column = clean_entity_column)
all_data <-
.clean_org_usa(data = all_data,
col = "nameOfficeAward",
clean_entity_column = clean_entity_column)
all_data <-
.clean_organization(data = all_data,
clean_entity_column = clean_entity_column,
col = "nameVendor")
all_data <-
.clean_organization(data = all_data,
clean_entity_column = clean_entity_column,
col = "nameVendorParent")
}
if (all_data %>% hasName("yearObligation")) {
all_data <- all_data %>%
rename(yearFiscalObligation = yearObligation)
}
all_data <-
all_data %>%
select(-matches("slugDUNS|dateContractSigned")) %>%
select(one_of(c("idContract", "dateObligation")), everything())
if (clean_entity_column) {
all_data <-
.clean_organization(data = all_data, col = "nameVendor")
all_data <-
.clean_organization(data = all_data, col = "nameVendorParent")
}
if (unformat) {
all_data <-
all_data %>%
mutate_if(is.numeric, as.numeric)
}
all_data <-
all_data %>%
arrange(dateObligation)
if (decode_contract_ids) {
all_data <-
tbl_decode_federal_contract_ids(data = all_data, use_sam = F)
}
if (snake_names) {
all_data <-
all_data %>%
janitor::clean_names()
}
all_data
}
#' Clean FPDS Agencies
#'
#' Cleans malformed FPDS
#' Agencies and brings in parents.
#'
#' @param data a \code{tibble}
#'
#' @return
#' @export
#'
#' @examples
clean_fpds_agencies <-
function(data) {
dict_agencies <- dictionary_government_agencies()
all_names <- names(data)
if (data %>% hasName("nameAgencyAward")) {
df_distinct <-
data %>%
filter(!is.na(idAgencyAward)) %>%
distinct(idAgencyAward, nameAgencyAward)
df_distinct <-
df_distinct %>%
left_join(
dict_agencies %>%
select(
nameAgencyAwardParent = nameAgency,
idAgencyAward = idOffice,
nameAgencyAwardNew = nameOffice
),
by = "idAgencyAward"
) %>%
group_by(idAgencyAward) %>%
dplyr::slice(1) %>%
ungroup()
df_distinct <- df_distinct %>%
mutate(nameAgencyAward = ifelse(
is.na(nameAgencyAwardNew),
nameAgencyAward,
nameAgencyAwardNew
)) %>%
select(-nameAgencyAwardNew)
data <-
data %>% select(-nameAgencyAward) %>%
left_join(df_distinct, by = "idAgencyAward") %>%
select(one_of(names(data)), everything())
}
if (data %>% hasName("nameAgencyAward")) {
df_distinct <-
data %>%
filter(!is.na(idAgencyAward)) %>%
distinct(idAgencyAward, nameAgencyAward)
df_distinct <-
df_distinct %>%
left_join(
dict_agencies %>%
select(
nameAgencyAwardParent = nameAgency,
idAgencyAward = idOffice,
nameAgencyAwardNew = nameOffice
),
by = "idAgencyAward"
) %>%
group_by(idAgencyAward) %>%
dplyr::slice(1) %>%
ungroup()
df_distinct <-
df_distinct %>%
mutate(nameAgencyAward = ifelse(
is.na(nameAgencyAwardNew),
nameAgencyAward,
nameAgencyAwardNew
)) %>%
select(-nameAgencyAwardNew)
data <-
data %>% select(-nameAgencyAward) %>%
left_join(df_distinct, by = "idAgencyAward") %>%
select(one_of(names(data)), everything())
}
if (data %>% hasName("nameAgencyFunding")) {
df_distinct <-
data %>%
filter(!is.na(idAgencyFunding)) %>%
distinct(idAgencyFunding, nameAgencyFunding)
df_distinct <-
df_distinct %>%
left_join(
dict_agencies %>%
select(
nameAgencyFundingParent = nameAgency,
idAgencyFunding = idOffice,
nameAgencyFundingNew = nameOffice
),
by = "idAgencyFunding"
) %>%
group_by(idAgencyFunding) %>%
dplyr::slice(1) %>%
ungroup()
df_distinct <-
df_distinct %>%
mutate(nameAgencyFunding = ifelse(
is.na(nameAgencyFundingNew),
nameAgencyFunding,
nameAgencyFundingNew
)) %>%
select(-nameAgencyFundingNew)
data <-
data %>% select(-nameAgencyFunding) %>%
left_join(df_distinct, by = "idAgencyFunding") %>%
select(one_of(names(data)), everything())
}
if (data %>% hasName("nameAgencyAwardIDV")) {
df_distinct <-
data %>%
filter(!is.na(idAgencyAwardIDV)) %>%
distinct(idAgencyAwardIDV, nameAgencyAwardIDV)
df_distinct <-
df_distinct %>%
left_join(
dict_agencies %>%
select(
nameAgencyAwardIDVParent = nameAgency,
idAgencyAwardIDV = idOffice,
nameAgencyAwardIDVNew = nameOffice
),
by = "idAgencyAwardIDV"
) %>%
group_by(idAgencyAwardIDV) %>%
dplyr::slice(1) %>%
ungroup()
df_distinct <-
df_distinct %>%
mutate(nameAgencyAwardIDV = ifelse(
is.na(nameAgencyAwardIDVNew),
nameAgencyAwardIDV,
nameAgencyAwardIDVNew
)) %>%
select(-nameAgencyAwardIDVNew)
data <-
data %>% select(-nameAgencyAwardIDV) %>%
left_join(df_distinct, by = "idAgencyAwardIDV") %>%
select(one_of(names(data)), everything())
}
data
}
.parse_atom_xml_simple <-
function(url) {
doc <- read_xml(url)
xml_nodes <-
doc %>%
xml_contents()
df <- tibble(
column = xml_nodes %>% as_list() %>% unlist() %>% names() %>% str_remove_all(
"content.award.|content.IDV.|awardID.|relevantContractDates.|dollarValues.|totalDollarValues.|purchaserInformation.|contractData.|contractMarketingData.|ccrRegistrationDetails.|legislativeMandates.|vendor.vendorSiteDetails.|productOrServiceInformation.|listOfTreasuryAccounts.treasuryAccount.|vendorSocioEconomicIndicators.| vendor.vendorHeader.|vendorLineOfBusiness.|vendorBusinessTypes.|minorityOwned.|preferencePrograms.|placeOfPerformance.principalPlaceOfPerformance.|competition.|vendorRelationshipWithFederalGovernment.|typeOfGovernmentEntity.|vendorOrganizationFactors.|vendorCertifications.|treasuryAccountSymbol.|vendor.vendorHeader.|localGovernment.|federalGovernment.|businessOrOrganizationType.|profitStructure.|typeOfEducationalEntity.| placeOfPerformance.|contentOtherTransactionIDVcontractDetail.|\\contentIDVtransactionInformation|\\contractID"
),
value = xml_nodes %>% as_list() %>% unlist() %>% as.character()
)
df <-
df %>%
filter(column != "") %>%
mutate(idColumn = 1:n()) %>%
mutate(column = column %>% str_remove_all("\\."))
start_col <- df %>% slice(1) %>% select(column) %>% pull()
df_ids <-
df %>% filter(column == start_col) %>%
mutate(numberTransaction = 1:n()) %>%
select(numberTransaction, everything())
df <-
df %>%
left_join(df_ids %>% select(idColumn, numberTransaction)
, by = "idColumn") %>%
fill(numberTransaction) %>%
select(numberTransaction, everything())
data <-
df %>%
mutate(
column = column %>% str_remove_all(
"contractID| contentOtherTransactionIDVcontractDetail|contentOtherTransactionIDV| ^contractDetail"
)
) %>%
group_by(numberTransaction, idColumn, column) %>%
filter(idColumn == min(idColumn)) %>%
ungroup() %>%
select(numberTransaction, column, value)
col_order <- data$column %>% unique()
data <- data %>%
spread(column, value) %>%
select(one_of(col_order))
if (data %>%
hasName("vendorDUNSInformationvendorName")) {
data <-
data %>% rename(nameVendorDUNS = vendorDUNSInformationvendorName)
}
names(data) <-
names(data) %>% str_remove_all("contentOtherTransactionAward|contentOtherTransactionIDV|^contractID")
data <-
data %>%
dplyr::select(-matches("^generic"))
data <- .munge_fpds_names(data)
col_ids <-
tibble(name = names(data)) %>%
mutate(column = 1:n()) %>%
group_by(name) %>%
filter(column == min(column)) %>%
ungroup() %>%
pull(column)
data <- data[, col_ids]
data <- data %>%
.fix_contract_description() %>%
.fix_contract_name()
data <-
data %>%
mutate(urlFPDSContractAtom = url)
data
}
#' FPDS Special Search
#'
#' @param ...
#'
#' @return
#' @export
#'
#' @examples
fpds_s <-
function(snake_names = F,
unformat = T,
clean_entity_column = F,
exclude_bloat = F,
munge_tbl = T,
return_message = T,
...) {
d <- fpds_atom(...) %>% nrow() * 10
if (d < 15000) {
data <-
.fpds_single_csv_search(return_message = return_message, ...)
} else {
df_dates <- .generate_search_dates()
data <-
1:nrow(df_dates) %>%
map_dfr(function(x) {
df_date <- df_dates %>% dplyr::slice(x)
.fpds_csv_safe <-
possibly(.fpds_single_csv_search, tibble())
data <-
.fpds_csv_safe(
signed_date = c(df_date$start_date, df_date$end_date),
return_message = return_message,
...
)
}) %>%
mutate_if(is.numeric, as.numeric)
}
if (clean_entity_column) {
data <-
.clean_organization(data = data, col = "nameVendor")
data <-
.clean_organization(data = data, col = "nameVendorParent")
}
if (data %>% hasName("dateContractSigned")) {
data <-
data %>%
rename(dateObligation = dateContractSigned)
}
if (!munge_tbl) {
return(data)
}
data <-
data %>%
arrange(dateObligation)
data <-
data %>%
munge_lite(
snake_names = snake_names,
unformat = unformat,
exclude_bloat = exclude_bloat
)
data
}
#' Parse atom URLS simple form
#'
#' @param urls vector of urls
#'
#' @return
#' @export
#'
#' @examples
parse_fpds_atom_urls_simple <-
function(urls, exclude_bloat = F) {
urls %>%
future_map_dfr(function(x) {
.parse_atom_xml_simple(url = x)
}) %>%
.munge_data(exclude_bloat = exclude_bloat)
}
# xml ---------------------------------------------------------------------
.parse_content_nodes_detailed <-
function(content_nodes) {
all_data <-
seq_along(content_nodes) %>%
map_dfr(function(y) {
content_node <- content_nodes[[y]]
items <- names(as_list(content_node) %>% unlist())
values <-
as_list(content_node) %>% unlist() %>% as.character()
column <- xml_name(content_node)
#column %>% message()
double_nodes <-
c(
"awardID",
"contractID",
"agencySpecificAwardElements",
"OtherTransactionAwardID"
)
if (column %in% double_nodes) {
items <-
content_node %>% xml_contents() %>% xml_contents()
if (column == "OtherTransactionAwardID") {
column <- "awardID"
}
values <-
items %>% xml_text()
}
single_nodes <-
c(
"relevantContractDates",
"dollarValues",
"totalDollarValues",
"purchaserInformation",
"contractMarketingData",
"contractData",
"legislativeMandates",
"productOrServiceInformation",
"preferencePrograms",
"competition",
"transactionInformation",
"genericTags"
)
if (column %in% c("vendor", "contractDetail")) {
values <-
content_node %>% xml_contents() %>% .[[2]] %>% xml_contents() %>% xml_contents() %>% xml_text()
items <-
content_node %>% xml_contents() %>% .[[2]] %>% xml_contents() %>% xml_contents() %>% xml_name()
descriptions <-
content_node %>% xml_contents() %>% .[[2]] %>% xml_contents() %>% xml_contents() %>% xml_attrs() %>% as.character()
df <-
tibble(nameItem = items,
value = values,
description = descriptions)
items_2 <-
content_node %>% xml_contents() %>% xml_contents() %>% xml_contents() %>%
xml_contents() %>%
xml_name()
values_2 <-
content_node %>% xml_contents() %>% xml_contents() %>% xml_contents() %>%
xml_contents() %>%
xml_text()
descriptions_2 <-
content_node %>% xml_contents() %>% xml_contents() %>%
xml_contents() %>%
xml_contents() %>% xml_attrs() %>% as.character()
df_append <-
tibble(nameItem = items_2,
value = values_2,
description = descriptions_2) %>%
filter(nameItem != "text")
if (column == "contractDetail") {
column <- "vendor"
}
df <- df %>%
bind_rows(df_append) %>%
mutate(numberNode = y,
nameNode = column) %>%
select(numberNode, nameNode, everything())
return(df)
}
if (column == "placeOfPerformance") {
cn <- content_node %>% xml_contents()
df <-
seq_along(cn) %>%
map_dfr(function(x) {
values <-
cn[[x]] %>% xml_children() %>% xml_text()
if (length(values) == 0) {
items <- cn[[x]] %>% xml_name()
values <- cn[[x]] %>% xml_text()
} else {
items <-
cn[[x]] %>% xml_children() %>% xml_name()
}
tibble(nameItem = items, value = values)
})
df <-
df %>%
mutate(
description = "character(0)",
numberNode = y,
nameNode = column
) %>%
select(numberNode, nameNode, everything())
return(df)
}
if (!column %in% double_nodes) {
items <-
content_node %>% xml_contents()
values <-
items %>% xml_text()
}
item_names <- items %>% map(xml_name) %>% flatten_chr()
item_attributes <- items %>% map(xml_attrs)
descriptions <-
seq_along(item_attributes) %>%
map(function(x) {
values <-
item_attributes[[x]] %>% as.character() %>% str_c(collapse = " | ")
if (length(values) == 0) {
return(NA_character_)
}
values %>% gsub("\\s+", " ", .)
}) %>%
flatten_chr()
df_items <-
tibble(nameItem = item_names, description = descriptions)
df_items %>%
mutate(
value = str_trim(values),
description = descriptions,
numberNode = y,
nameNode = column
) %>%
select(numberNode, nameNode, everything())
})
all_data <-
all_data %>%
mutate(description = ifelse(description == "character(0)", NA, description)) %>%
mutate(idRow = 1:n())
df_values <-
all_data %>%
mutate(nameItem = ifelse(nameItem == nameNode, nameItem, str_c(nameNode, nameItem))) %>%
mutate(nameItem = nameItem %>% str_remove_all("\\-")) %>%
select(nameItem, value) %>%
filter(value != "")
df_order <-
df_values %>%
mutate(idRow = 1:n()) %>%
select(idRow, nameItem)
df_values <-
df_values %>%
select(nameItem, value) %>%
group_by(nameItem) %>%
dplyr::slice(1) %>%
ungroup() %>%
spread(nameItem, value)
df_descriptions <-
all_data %>% filter(!is.na(description)) %>%
mutate(nameItem = ifelse(
nameItem == nameNode,
nameItem,
str_c(nameNode, nameItem, "Description")
)) %>%
mutate(nameItem = nameItem %>% str_remove_all("\\-"))
df_descriptions <-
df_descriptions %>%
select(idRow, nameItem, description) %>%
filter(!is.na(description))
df_order <-
df_order %>%
bind_rows(df_descriptions %>% select(idRow, nameItem)) %>%
arrange(idRow)
df_descriptions <-
df_descriptions %>%
select(-idRow) %>%
group_by(nameItem) %>%
dplyr::slice(1) %>%
spread(nameItem, description)
data <-
df_values %>% bind_cols(df_descriptions) %>%
select(one_of(df_order$nameItem), everything()) %>%
mutate_if(is.character,
list(function(x) {
x %>% gsub("\\s+", " ", .)
})) %>%
mutate_if(is.numeric, as.numeric)
data
}
.parse_xml_entry <- function(xml_entry, use_detailed = F) {
xml_entry_nodes <- xml_contents(xml_entry)
xml_entry_names <- xml_name(xml_entry_nodes)
seq_along(xml_entry_names) %>%
map(function(x) {
table <-
xml_entry_names[[x]]
if (table == "title") {
title <- xml_entry_nodes[1] %>% xml_text()
df <- tibble(nameContract = title)
return(df)
}
if (table == "link") {
url <-
xml_entry_nodes[[x]] %>% html_attr("href")
df <- tibble(urlFPDSContractAtom = url)
return(df)
}
if (table == "modified") {
value <-
xml_entry_nodes[x] %>% xml_text()
if (value == "") {
return(invisible())
}
value <- value %>% lubridate::ymd_hms()
df <- tibble(datetimeContractModified = value)
return(df)
}
xml_part <- xml_entry_nodes[x]
values <- xml_part %>% as_list()
value <- values %>% unlist() %>% as.character()
items <- values %>% unlist() %>% names()
items <-
seq_along(items) %>%
map_chr(function(x) {
parts <-
items[[x]] %>%
str_split("\\.") %>% flatten_chr() %>%
str_remove_all("NASASpecificAwardElements")
start <- length(parts) - 1
end <- length(parts)
parts <-
parts[start:end] %>% str_c(collapse = "")
parts %>% substr(1, 1) %>% str_to_lower() %>%
str_c(.,
parts %>% substr(2, nchar(parts)), collapse = "")
})
df <-
tibble(item = items, value)
df <- df %>% select(item, value)
df <-
df %>% left_join(dictionary_fpds_names() %>% rename(item = nameFPDS),
by = "item")
if (df %>% filter(is.na(nameActual)) %>% nrow() > 0) {
missing_items <-
df %>% filter(is.na(nameActual)) %>% pull(item)
seq_along(missing_items) %>%
map(function(x) {
glue("Missing {missing_items[[x]]}") %>% message()
})
df <- df %>%
mutate(nameActual = case_when(is.na(nameActual) ~ item,
TRUE ~ nameActual))
}
df <-
df %>%
select(nameActual, value)
df <-
df %>%
group_by(nameActual) %>%
dplyr::slice(1) %>%
ungroup()
col_order <- df$nameActual
df %>%
spread(nameActual, value) %>% select(one_of(col_order)) %>%
mutate_if(is.numeric, as.numeric)
}) %>% reduce(bind_cols)
}
.parse_fpds_atom_url <-
function(url = "https://www.fpds.gov/ezsearch/FEEDS/ATOM?FEEDNAME=PUBLIC&q=GLOBAL_VENDOR_NAME:PALANTIR&sortBy=SIGNED_DATE&desc=Y&start=520&sortBy=SIGNED_DATE",
keep_key_columns = F,
return_message = T) {
if (return_message) {
message(str_c("Parsing: ", url))
}
doc <-
read_xml(url)
xml_nodes <-
xml_contents(doc)
tables <-
xml_name(xml_nodes)
df_table_nodes <-
tibble(table = tables) %>%
mutate(idTable = 1:n())
df_ids <-
filter(.data = df_table_nodes, table == "entry")
data <-
seq_len(nrow(df_ids)) %>%
map_dfr(function(node_x) {
df_row <-
df_ids[node_x, ]
table <- df_row$table
idTable <- df_row$idTable
if (table == "title") {
title <-
xml_contents(doc) %>% xml_text() %>% .[[1]] %>% str_to_upper() %>% gsub("\\s+", " ", .)
df <- tibble(titleSearch = title)
return(df)
}
if (table == "link") {
url <-
xml_nodes[[idTable]] %>% html_attr("href")
df <- tibble(urlFPDSAtom = url)
return(df)
}
if (table == "modified") {
value <- xml_nodes[[idTable]] %>% xml_text()
if (value == "") {
return(invisible())
}
}
if (table == "author") {
value <- xml_nodes[[idTable]] %>% xml_text()
if (value == "") {
return(invisible())
}
}
xml_entry <-
xml_nodes[[idTable]]
df <-
.parse_xml_entry_new(xml_entry = xml_entry)
if (df %>% hasName("idContractIDV") &
df %>% hasName("idAgencyAward")) {
award <- unique(df$idAgencyAward) %>% str_c(collapse = "|")
award <- glue("^{award}") %>% as.character()
df <- df %>%
mutate(idContractIDV = idContractIDV %>% str_remove_all(award))
}
if (df %>% hasName("codeModificationIDV") &&
df %>% hasName("idContractIDV")) {
idvmod <-
unique(df$codeModificationIDV) %>% str_c(collapse = "|")
idvmod <- as.character(glue("{idvmod}$"))
df <- df %>%
mutate(idContractIDV = idContractIDV %>% str_remove_all(idvmod))
}
df
})
df_double <-
tibble(name = names(data)) %>% count(name, sort = T) %>% filter(n > 1)
if (nrow(df_double) > 0) {
df_double$name %>% str_c(collapse = '\n') %>% message()
}
data <-
select(.data = data, -matches("remove"))
if (!data %>%
hasName("dateContractSigned") &
data %>% hasName("datetimeContractSigned")) {
data <-
data %>%
rename(dateContractSigned = datetimeContractSigned)
}
start_1 <-
data %>%
select(
matches(
"idDUNS|nameVendor|nameAgency|nameOffice|descriptionOfficeFundingRequest|descriptionObligation|acronymProgram|nameMajorProgram"
)
) %>% names()
start_2 <-
data %>%
select(matches("amount|^count[A-Z]")) %>%
select(-matches("country")) %>%
names()
one_names <- data %>% select(ends_with("1")) %>% names()
if (length(one_names) > 0) {
data <- data %>% select(-one_of(one_names))
}
sel_col <- names(data)[names(data) %in% c(
"nameContract",
"idContract",
"dateContractSigned",
"dateContractEffective",
"idAgencyFunding"
)]
data <-
data %>%
mutate(urlFPDSAtom = url) %>%
select(one_of(sel_col),
matches("idDUNS"),
one_of(c(start_1, start_2)),
everything())
if (data %>% hasName("genericString06")) {
data <- data %>%
select(-genericString06)
}
if (keep_key_columns) {
data <-
select_key_columns(data)
}
data
}
.parse_xml_fpds_new <- function(url = "https://www.fpds.gov/ezsearch/FEEDS/ATOM?FEEDNAME=PUBLIC&q=GLOBAL_VENDOR_NAME:PALANTIR&sortBy=SIGNED_DATE&desc=Y&start=520&sortBy=SIGNED_DATE", keep_key_columns = F, return_message = F) {
if (return_message) {
message(str_c("Parsing: ", url))
}
doc <- read_xml(url)
xml_ns_strip(doc)
entries <- doc %>% xml_find_all("//entry")
data <-
seq_along(entries) %>%
map_dfr(function(x){
.parse_xml_entry_new(xml_entry = entries[[x]])
})
df_double <-
tibble(name = names(data)) %>% count(name, sort = T) %>% filter(n > 1)
if (nrow(df_double) > 0) {
df_double$name %>% str_c(collapse = '\n') %>% message()
}
data <-
select(.data = data, -matches("remove"))
if (!data %>%
hasName("dateContractSigned") &
data %>% hasName("datetimeContractSigned")) {
data <-
data %>%
rename(dateContractSigned = datetimeContractSigned)
}
start_1 <-
data %>%
select(
matches(
"idDUNS|nameVendor|nameAgency|nameOffice|descriptionOfficeFundingRequest|descriptionObligation|acronymProgram|nameMajorProgram"
)
) %>% names()
start_2 <-
data %>%
select(matches("amount|^count[A-Z]")) %>%
select(-matches("country")) %>%
names()
one_names <- data %>% select(ends_with("1")) %>% names()
if (length(one_names) > 0) {
data <- data %>% select(-one_of(one_names))
}
sel_col <- names(data)[names(data) %in% c(
"nameContract",
"idContract",
"dateContractSigned",
"dateContractEffective",
"idAgencyFunding"
)]
data <-
data %>%
mutate(urlFPDSAtom = url) %>%
select(one_of(sel_col),
matches("idDUNS"),
one_of(c(start_1, start_2)),
everything())
if (keep_key_columns) {
data <-
select_key_columns(data)
}
data
}
.parse_fpds_atom_urls_multi <-
function(urls = c(
"http://api.gdeltproject.org/api/v2/doc/doc?query=brooklyn%20nets%20sourcelang:english&mode=artlist&format=json×pan=12w&maxrecords=250&sort=datedesc",
"http://api.gdeltproject.org/api/v2/doc/doc?query=domain:netsdaily.com%20sourcelang:english&mode=artlist&format=json×pan=12w&maxrecords=250&sort=datedesc",
"http://api.gdeltproject.org/api/v2/doc/doc?query=theme:econ_bitcoin%20sourcelang:english&mode=artlist&format=json×pan=12w&maxrecords=250&sort=datedesc"
),
snake_names = T,
clean_entity_column = F,
unformat = T,
exclude_bloat = T,
clean_address = T,
return_message = T,
keep_key_columns = TRUE) {
all_data <-
tibble()
.parse_xml_fpds_new_safe <-
purrr::possibly(.parse_xml_fpds_new, tibble())
success <- function(res) {
data <-
res$url %>%
.parse_xml_fpds_new_safe(keep_key_columns = keep_key_columns)
all_data <<-
all_data %>%
bind_rows(data)
}
failure <- function(msg) {
tibble()
}
urls %>%
walk(function(x) {
curl_fetch_multi(url = x, success, failure)
})
multi_run()
if (return_message) {
message("\n\nData acquired\n\n")
}
if (all_data %>% hasName("zipcodePerformancePerformance") &&
all_data %>% hasName("zipcodePerformance")) {
all_data <- all_data %>%
mutate(
zipcodePerformance = case_when(
is.na(zipcodePerformance) ~ zipcodePerformancePerformance,
TRUE ~ zipcodePerformance
)
) %>%
select(-zipcodePerformancePerformance)
}
if (all_data %>% hasName("nameVendorParent") &
all_data %>% hasName("nameVendor")) {
all_data <- all_data %>%
mutate(
nameVendor = case_when(
is.na(nameVendor) & !is.na(nameVendorParent) ~ nameVendorParent,
is.na(nameVendor) &
is.na(nameVendor) ~ "UNDISCLOSED VENDOR",
TRUE ~ nameVendor
),
nameVendorParent = case_when(
is.na(nameVendorParent) & !is.na(nameVendor) ~ nameVendorParent,
is.na(nameVendor) &
is.na(nameVendor) ~ "UNDISCLOSED VENDOR PARENT",
TRUE ~ nameVendorParent
)
) %>%
mutate_at(c("idDUNS", "idDUNSParent"),
list(function(x) {
ifelse(x == 0, NA_integer_, x)
})) %>%
mutate(hasContractParent = idDUNSParent != idDUNS) %>%
mutate_if(is.numeric, as.numeric)
}
if (all_data %>% hasName("nameVendorParent") &
!all_data %>% hasName("nameVendor")) {
all_data <- all_data %>%
mutate(nameVendor = nameVendorParent)
}
if (all_data %>% hasName("idContractIDV")) {
all_data <- all_data %>%
mutate(isIDV = !is.na(idContractIDV))
}
if (all_data %>% hasName("idCGAC") &
all_data %>% hasName("codeAccountOMB")) {
all_data <-
.generate_federal_account_ids(data = all_data,
cgac_column = "idCGAC",
account_column = "codeAccountOMB")
}
all_data <-
all_data %>%
.fix_contract_name() %>%
.fix_contract_description()
names(all_data) <-
names(all_data) %>% str_replace_all("PerformancePerformance", "Performance")
if (all_data %>% hasName("zipcodePerformance") &
all_data %>% hasName("locationPerformance")) {
all_data <- all_data %>%
mutate(zipcodePerformance = case_when(
is.na(zipcodePerformance) ~ locationPerformance,
TRUE ~ zipcodePerformance
)) %>%
select(-locationPerformance)
}
change_v <- all_data %>% hasName("nameContractor") &&
all_data %>% hasName("nameVendor")
if (change_v) {
all_data <- all_data %>%
mutate(isContractorNotVendor = nameContractor != nameVendor)
}
if (all_data %>% hasName("dateContractSigned")) {
all_data <-
mutate(all_data, dateObligation = dateContractSigned %>% as.Date())
} else {
if (all_data %>% hasName("datetimeTransactionCreated")) {
all_data <-
all_data %>%
mutate(dateObligation = as.Date(datetimeTransactionCreated))
}
if (all_data %>% hasName("datetimeContractModified")) {
all_data <- all_data %>%
mutate(dateObligation = as.Date(datetimeContractModified))
}
}
all_data <-
select(
all_data,
one_of("dateObligation"),
matches("idContract"),
matches("dateContractEffective"),
one_of("nameContract"),
matches("description"),
matches("datetimeContractModified"),
matches("idDUNS|Vendor"),
matches("date"),
matches("amount"),
matches("Department"),
matches("Agency"),
matches("Office"),
matches("NAICS"),
matches("ProductService"),
everything()
)
if (all_data %>% hasName("dateContractEffective") &&
all_data %>% hasName("dateObligation")) {
all_data <- all_data %>%
mutate(
countDaysSinceContractEffective = (dateObligation - dateContractEffective) %>% as.numeric(),
isFutureObligation = countDaysSinceContractEffective < 0
)
}
if (all_data %>% hasName("dateContractCompletionCurrent") &
all_data %>% hasName("dateContractCompletionUltimate")) {
all_data <- all_data %>%
mutate(hasModifiedContractDate = dateContractCompletionUltimate != dateContractCompletionCurrent)
}
dt <-
all_data %>% select(matches("datetimeReg|datetimeRen")) %>% names()
if (length(dt) > 0) {
all_data <- mutate_at(all_data, dt, as.Date)
new <-
names(all_data)[names(all_data) %in% dt] %>% str_remove_all("time")
names(all_data)[names(all_data) %in% dt] <- new
}
if (clean_entity_column) {
all_data <-
.clean_org_usa(data = all_data,
col = "nameAgencyAward",
clean_entity_column = clean_entity_column)
all_data <-
.clean_org_usa(data = all_data,
col = "nameAgencyFunding",
clean_entity_column = clean_entity_column)
all_data <-
.clean_org_usa(data = all_data,
col = "nameOfficeAward",
clean_entity_column = clean_entity_column)
all_data <-
.clean_org_usa(data = all_data,
col = "nameOfficeFunding",
clean_entity_column = clean_entity_column)
all_data <-
.clean_organization(data = all_data,
clean_entity_column = clean_entity_column,
col = "nameVendor")
all_data <-
.clean_organization(data = all_data,
clean_entity_column = clean_entity_column,
col = "nameContractor")
all_data <-
.clean_organization(data = all_data,
clean_entity_column = clean_entity_column,
col = "nameVendorParent")
}
all_data <- mutate_if(all_data, is.numeric, as.numeric)
all_data <-
munge_data(
data = all_data,
unformat = unformat,
exclude_bloat = exclude_bloat,
clean_address = clean_address,
snake_names = F
)
if (all_data %>% hasName("idContractAnalysis") &&
all_data %>% hasName("idContractIDV")) {
all_data <- all_data %>%
mutate(idContractAnalysis = case_when(
is.na(idContractAnalysis) ~ idContractIDV,
TRUE ~ idContractAnalysis
))
}
if (all_data %>% hasName("idContract") &
all_data %>% hasName("idAward")) {
all_data <-
all_data %>%
mutate(idContract = ifelse(is.na(idContract), idAward, idContract)) %>%
select(-idAward)
}
if ((!all_data %>% hasName("idContract")) &
(all_data %>% hasName("idAward"))) {
all_data <-
all_data %>%
rename(idContract = idAward)
}
all_data <-
unique(all_data) %>%
.remove_na()
if (!all_data %>% hasName("nameNAICS") &
all_data %>% hasName("idNAICS")) {
df_naics <-
all_data %>%
select(idNAICS) %>%
filter(!is.na(idNAICS)) %>%
distinct()
df_naics <- df_naics %>%
left_join(dictionary_naics_codes(),
by = "idNAICS")
all_data <-
all_data %>%
select(-nameNAICS) %>%
left_join(df_naics, by = "idNAICS")
df_naics_names <-
all_data %>%
group_by(idNAICS, nameNAICS) %>%
summarise(amount = sum(amountObligation, na.rm = T)) %>%
filter(amount == max(amount)) %>%
dplyr::slice(1) %>%
select(idNAICS, nameNAICS) %>%
ungroup()
all_data <-
all_data %>%
select(-nameNAICS) %>%
left_join(df_naics_names, by = "idNAICS") %>%
select(names(all_data), everything())
}
if (!all_data %>% hasName("nameProductService") &
all_data %>% hasName("codeProductService")) {
df_psc <-
all_data %>%
filter(!is.na(codeProductService)) %>%
distinct(codeProductService)
df_psc <-
df_psc %>%
left_join(
dictionary_psc_active() %>% select(typePSC,
codeProductService,
nameProductService) %>% mutate_all(str_to_upper)
,
by = "codeProductService"
)
all_data <-
all_data %>%
select(-one_of("nameProductService")) %>%
left_join(df_psc, by = "codeProductService")
all_data <- add_solicitation_group(data = all_data)
df_psc_names <-
all_data %>%
group_by(codeProductService, nameProductService) %>%
summarise(amount = sum(amountObligation, na.rm = T)) %>%
filter(amount == max(amount)) %>%
dplyr::slice(1) %>%
select(codeProductService, nameProductService) %>%
ungroup()
all_data <- all_data %>%
select(-nameProductService) %>%
left_join(df_psc_names, by = "codeProductService") %>%
select(names(all_data), everything())
}
if (all_data %>% hasName("nameVendor") &
all_data %>% hasName("nameVendorParent")) {
all_data <- all_data %>%
mutate(nameVendor = case_when(is.na(nameVendor) ~ nameVendor,
TRUE ~ nameVendorParent))
}
if (all_data %>% hasName("typeResearch")) {
all_data <- all_data %>%
mutate(
isPossibleSBIRSTTR = descriptionObligation %>% str_detect(
"(PHASE 1)|(PHASE I)|(PHASE II)|(PHASE 2)|(PHASE III)|STTR|SBIR"
),
isFilteredSBIR = case_when(
typeResearch %in%
c(
"SMALL BUSINESS INNOVATION RESEARCH PROGRAM PHASE I ACTION",
"SMALL BUSINESS INNOVATION RESEARCH PROGRAM PHASE II ACTION",
"SMALL TECHNOLOGY TRANSFER RESEARCH PROGRAM PHASE I",
"SMALL TECHNOLOGY TRANSFER RESEARCH PROGRAM PHASE II"
) ~ T,
is.na(typeResearch) & isPossibleSBIRSTTR ~ T,
TRUE ~ F
)
)
}
if (all_data %>% hasName("nameFirstInvestigator") &
all_data %>% hasName("nameLastInvestigator")) {
df_pis <-
all_data %>%
distinct(nameFirstInvestigator, nameLastInvestigator) %>%
filter(!is.na(nameFirstInvestigator)) %>%
filter(!nameFirstInvestigator %>% str_detect("UNKNOWN")) %>%
filter(!is.na(nameLastInvestigator) %>% str_detect("UNKNOWN"))
if (nrow(df_pis) > 0) {
df_pis <-
df_pis %>%
mutate(
namePrincipalInvestigator = case_when(
nameFirstInvestigator %>% str_detect("UNKNOWN") ~ NA_character_,
TRUE ~ str_c(nameFirstInvestigator, nameLastInvestigator, sep = " ")
)
) %>%
filter(!is.na(namePrincipalInvestigator))
df_pis <- df_pis %>%
clean_entity_data(entity_column = "namePrincipalInvestigator") %>%
select(-namePrincipalInvestigator) %>%
rename(namePrincipalInvestigator = namePrincipalInvestigatorClean)
all_data <-
all_data %>%
left_join(df_pis,
by = c("nameFirstInvestigator", "nameLastInvestigator"))
}
}
## ENTITY RESOLUTION ### NEEDS FIXING
if (all_data %>% hasName("cageVendor")) {
df_cage <-
all_data %>%
select(idDUNSParent, cageVendor, amountObligation) %>%
filter(!is.na(cageVendor)) %>%
count(idDUNSParent, cageVendor, wt = amountObligation) %>%
group_by(cageVendor) %>%
filter(n == max(n)) %>%
slice(1) %>%
ungroup() %>%
select(idDUNSParent, cageVendor)
all_data <-
all_data %>%
select(-cageVendor) %>%
left_join(df_cage, by = "idDUNSParent") %>%
distinct() %>%
select(one_of(names(all_data)), everything())
}
if (all_data %>% hasName("typeAward")) {
all_data <- all_data %>%
mutate(
typeProcurementFPDS = case_when(
typeAward %>% str_detect("BPA") ~ "BLANKET PURCHASE AGREEMENT",
TRUE ~ typeAward
),
isGrantType = typeAward %>% str_detect("GRANT")
)
}
if (all_data %>% hasName("yearObligation")) {
all_data <- all_data %>%
rename(yearFiscalObligation = yearObligation)
}
all_data <-
all_data %>%
select(-matches("slugDUNS|dateContractSigned")) %>%
select(one_of(c(
"idContractAnalysis", "idContract", "dateObligation"
)), everything())
name_cols <-
all_data %>% select(matches("nameAgency|nameOffice|nameDepartment")) %>% names()
name_cols %>%
walk(function(x) {
all_data <<- .clean_org_usa(data = all_data,
col = x,
clean_entity_column = clean_entity_column)
})
if (exclude_bloat) {
all_data <- all_data %>%
select(-matches("urlFPDS"))
}
all_data <- arrange(.data = all_data, dateObligation)
all_data
}
#' Parse ATOM FPDS urls
#'
#' @param urls vector of fpds URLS
#' @param return_message if \code{TRUE} return a message
#' @param clean_entity_column if \code{TRUE} cleans entity columns
#' @param unformat if \code{TRUE} unformats formattable columns
#' @param keep_key_columns if \code{TRUE} returns only key columns
#' @param exclude_bloat
#' @param snake_names
#' @param use_future
#' @param clean_address
#' @param show_progress
#'
#' @return \code{tibble}
#' @export
#'
#' @examples
#' @example inst/examples/parse_atom_urls.r
parse_fpds_atom_urls <-
function(urls = NULL,
use_future = T,
clean_entity_column = F,
unformat = T,
keep_key_columns = F,
exclude_bloat = T,
clean_address = T,
snake_names = F,
show_progress = T,
return_message = T) {
options(warn = -1)
if (length(urls) == 0) {
stop("Enter ATOM URLs")
}
.parse_fpds_atom_url_safe <-
possibly(.parse_fpds_atom_urls_multi, tibble())
if (use_future) {
options(future.globals.maxSize = 999999 * 1024 ^ 12)
all_data <-
urls %>%
future_map_dfr(function(url) {
.parse_fpds_atom_url_safe(urls = url,
snake_names = F,
clean_entity_column = clean_entity_column,
unformat = unformat,
exclude_bloat = exclude_bloat,
clean_address = clean_address,
return_message = F,
keep_key_columns = keep_key_columns)
})
} else {
all_data <-
urls %>%
map_dfr(function(url) {
.parse_fpds_atom_url_safe(urls = url,
snake_names = F,
clean_entity_column = clean_entity_column,
unformat = unformat,
exclude_bloat = exclude_bloat,
clean_address = clean_address,
return_message = F,
keep_key_columns = keep_key_columns)
})
}
if (nrow(all_data) == 0) {
"No data" %>% message()
return(tibble())
}
actions <- nrow(all_data) %>% comma(digits = 0)
contracts <-
all_data %>% distinct(idContractAnalysis) %>% nrow() %>% comma(digits = 0)
duns <-
all_data %>% distinct(idDUNS) %>% nrow() %>% comma(digits = 0)
parent_duns <-
all_data %>% distinct(idDUNSParent) %>% nrow() %>% comma(digits = 0)
amt <-
all_data$amountObligation %>% sum() %>% currency(digits = 0)
from_date <- all_data$dateObligation %>% min(na.rm = T)
to_date <- all_data$dateObligation %>% max(na.rm = T)
glue(
"\n\n{green({amt})} procured between {red({from_date})} and {red({to_date})} across {yellow(actions)} actions amongst {magenta(contracts)} contracts allocated to {cyan(duns)} distinct DUNS and {blue(parent_duns)} distinct parent DUNS\n\n"
) %>% message()
all_data <- arrange(.data = all_data, dateObligation)
if (snake_names) {
all_data <-
all_data %>%
clean_names()
}
all_data
}
parse_fpds_atom_urls_old <-
function(urls = NULL,
use_future = T,
clean_entity_column = F,
unformat = T,
keep_key_columns = F,
exclude_bloat = T,
clean_address = T,
snake_names = F,
show_progress = T,
return_message = T) {
options(warn = -1)
if (length(urls) == 0) {
stop("Enter ATOM URLs")
}
.parse_fpds_atom_url_safe <-
possibly(parse_fpds_atom_urls_multi, tibble())
if (use_future) {
options(future.globals.maxSize = 999999 * 1024 ^ 12)
all_data <-
urls %>%
future_map_dfr(function(url) {
.parse_fpds_atom_url_safe(
url = url,
keep_key_columns = keep_key_columns,
return_message = return_message
)
}, .progress = show_progress)
} else {
all_data <-
urls %>%
map_dfr(function(url) {
.parse_fpds_atom_url_safe(
url = url,
keep_key_columns = keep_key_columns,
return_message = return_message
)
})
}
if (nrow(all_data) == 0) {
"No data" %>% message()
return(tibble())
}
if (return_message) {
message("\n\nData acquired\n\n")
}
if (all_data %>% hasName("zipcodePerformancePerformance") &&
all_data %>% hasName("zipcodePerformance")) {
all_data <- all_data %>%
mutate(
zipcodePerformance = case_when(
is.na(zipcodePerformance) ~ zipcodePerformancePerformance,
TRUE ~ zipcodePerformance
)
) %>%
select(-zipcodePerformancePerformance)
}
if (all_data %>% hasName("nameVendorParent") &
all_data %>% hasName("nameVendor")) {
all_data <- all_data %>%
mutate(
nameVendor = case_when(
is.na(nameVendor) & !is.na(nameVendorParent) ~ nameVendorParent,
is.na(nameVendor) &
is.na(nameVendor) ~ "UNDISCLOSED VENDOR",
TRUE ~ nameVendor
),
nameVendorParent = case_when(
is.na(nameVendorParent) & !is.na(nameVendor) ~ nameVendorParent,
is.na(nameVendor) &
is.na(nameVendor) ~ "UNDISCLOSED VENDOR PARENT",
TRUE ~ nameVendorParent
)
) %>%
mutate_at(c("idDUNS", "idDUNSParent"),
list(function(x) {
ifelse(x == 0, NA_integer_, x)
})) %>%
mutate(hasContractParent = idDUNSParent != idDUNS) %>%
mutate_if(is.numeric, as.numeric)
}
if (all_data %>% hasName("nameVendorParent") &
!all_data %>% hasName("nameVendor")) {
all_data <- all_data %>%
mutate(nameVendor = nameVendorParent)
}
if (all_data %>% hasName("idContractIDV")) {
all_data <- all_data %>%
mutate(isIDV = !is.na(idContractIDV))
}
if (all_data %>% hasName("idCGAC") &
all_data %>% hasName("codeAccountOMB")) {
all_data <-
.generate_federal_account_ids(data = all_data,
cgac_column = "idCGAC",
account_column = "codeAccountOMB")
}
all_data <-
all_data %>%
.fix_contract_name() %>%
.fix_contract_description()
names(all_data) <-
names(all_data) %>% str_replace_all("PerformancePerformance", "Performance")
if (all_data %>% hasName("zipcodePerformance") &
all_data %>% hasName("locationPerformance")) {
all_data <- all_data %>%
mutate(zipcodePerformance = case_when(
is.na(zipcodePerformance) ~ locationPerformance,
TRUE ~ zipcodePerformance
)) %>%
select(-locationPerformance)
}
change_v <- all_data %>% hasName("nameContractor") &&
all_data %>% hasName("nameVendor")
if (change_v) {
all_data <- all_data %>%
mutate(isContractorNotVendor = nameContractor != nameVendor)
}
if (all_data %>% hasName("dateContractSigned")) {
all_data <- mutate(all_data, dateObligation = dateContractSigned %>% as.Date())
} else {
if (all_data %>% hasName("datetimeTransactionCreated")) {
all_data <-
all_data %>%
mutate(dateObligation = as.Date(datetimeTransactionCreated))
}
if (all_data %>% hasName("datetimeContractModified")) {
all_data <- all_data %>%
mutate(dateObligation = as.Date(datetimeContractModified))
}
}
all_data <-
select(
all_data,
one_of("dateObligation"),
matches("idContract"),
matches("dateContractEffective"),
one_of("nameContract"),
matches("description"),
matches("datetimeContractModified"),
matches("idDUNS|Vendor"),
matches("date"),
matches("amount"),
matches("Department"),
matches("Agency"),
matches("Office"),
matches("NAICS"),
matches("ProductService"),
everything()
)
if (all_data %>% hasName("dateContractEffective") && all_data %>% hasName("dateObligation")) {
all_data <- all_data %>%
mutate(
countDaysSinceContractEffective = (dateObligation - dateContractEffective) %>% as.numeric(),
isFutureObligation = countDaysSinceContractEffective < 0
)
}
if (all_data %>% hasName("dateContractCompletionCurrent") &
all_data %>% hasName("dateContractCompletionUltimate")) {
all_data <- all_data %>%
mutate(hasModifiedContractDate = dateContractCompletionUltimate != dateContractCompletionCurrent)
}
dt <-
all_data %>% select(matches("datetimeReg|datetimeRen")) %>% names()
if (length(dt) > 0) {
all_data <- mutate_at(all_data, dt, as.Date)
new <-
names(all_data)[names(all_data) %in% dt] %>% str_remove_all("time")
names(all_data)[names(all_data) %in% dt] <- new
}
if (clean_entity_column) {
all_data <-
.clean_org_usa(data = all_data,
col = "nameAgencyAward",
clean_entity_column = clean_entity_column)
all_data <-
.clean_org_usa(data = all_data,
col = "nameAgencyFunding",
clean_entity_column = clean_entity_column)
all_data <-
.clean_org_usa(data = all_data,
col = "nameOfficeAward",
clean_entity_column = clean_entity_column)
all_data <-
.clean_org_usa(data = all_data,
col = "nameOfficeFunding",
clean_entity_column = clean_entity_column)
all_data <-
.clean_organization(data = all_data,
clean_entity_column = clean_entity_column,
col = "nameVendor")
all_data <-
.clean_organization(data = all_data,
clean_entity_column = clean_entity_column,
col = "nameContractor")
all_data <-
.clean_organization(data = all_data,
clean_entity_column = clean_entity_column,
col = "nameVendorParent")
}
all_data <- mutate_if(all_data, is.numeric, as.numeric)
all_data <-
munge_data(
data = all_data,
unformat = unformat,
exclude_bloat = exclude_bloat,
clean_address = clean_address,
snake_names = F
)
if (all_data %>% hasName("idContractAnalysis") &&
all_data %>% hasName("idContractIDV")) {
all_data <- all_data %>%
mutate(idContractAnalysis = case_when(
is.na(idContractAnalysis) ~ idContractIDV,
TRUE ~ idContractAnalysis
))
}
if (all_data %>% hasName("idContract") &
all_data %>% hasName("idAward")) {
all_data <-
all_data %>%
mutate(idContract = ifelse(is.na(idContract), idAward, idContract)) %>%
select(-idAward)
}
if ((!all_data %>% hasName("idContract")) &
(all_data %>% hasName("idAward"))) {
all_data <-
all_data %>%
rename(idContract = idAward)
}
all_data <-
unique(all_data) %>%
.remove_na()
if (!all_data %>% hasName("nameNAICS") &
all_data %>% hasName("idNAICS")) {
df_naics <-
all_data %>%
select(idNAICS) %>%
filter(!is.na(idNAICS)) %>%
distinct()
df_naics <- df_naics %>%
left_join(dictionary_naics_codes(),
by = "idNAICS")
all_data <-
all_data %>%
select(-nameNAICS) %>%
left_join(df_naics, by = "idNAICS")
df_naics_names <-
all_data %>%
group_by(idNAICS, nameNAICS) %>%
summarise(amount = sum(amountObligation, na.rm = T)) %>%
filter(amount == max(amount)) %>%
dplyr::slice(1) %>%
select(idNAICS, nameNAICS) %>%
ungroup()
all_data <-
all_data %>%
select(-nameNAICS) %>%
left_join(df_naics_names, by = "idNAICS") %>%
select(names(all_data), everything())
}
if (!all_data %>% hasName("nameProductService") &
all_data %>% hasName("codeProductService")) {
df_psc <-
all_data %>%
filter(!is.na(codeProductService)) %>%
distinct(codeProductService)
df_psc <-
df_psc %>%
left_join(
dictionary_psc_active() %>% select(typePSC,
codeProductService,
nameProductService) %>% mutate_all(str_to_upper)
,
by = "codeProductService"
)
all_data <-
all_data %>%
select(-one_of("nameProductService")) %>%
left_join(df_psc, by = "codeProductService")
all_data <- add_solicitation_group(data = all_data)
df_psc_names <-
all_data %>%
group_by(codeProductService, nameProductService) %>%
summarise(amount = sum(amountObligation, na.rm = T)) %>%
filter(amount == max(amount)) %>%
dplyr::slice(1) %>%
select(codeProductService, nameProductService) %>%
ungroup()
all_data <- all_data %>%
select(-nameProductService) %>%
left_join(df_psc_names, by = "codeProductService") %>%
select(names(all_data), everything())
}
if (all_data %>% hasName("nameVendor") &
all_data %>% hasName("nameVendorParent")) {
all_data <- all_data %>%
mutate(nameVendor = case_when(is.na(nameVendor) ~ nameVendor,
TRUE ~ nameVendorParent))
}
if (all_data %>% hasName("typeResearch")) {
all_data <- all_data %>%
mutate(
isPossibleSBIRSTTR = descriptionObligation %>% str_detect(
"(PHASE 1)|(PHASE I)|(PHASE II)|(PHASE 2)|(PHASE III)|STTR|SBIR"
),
isFilteredSBIR = case_when(
typeResearch %in%
c(
"SMALL BUSINESS INNOVATION RESEARCH PROGRAM PHASE I ACTION",
"SMALL BUSINESS INNOVATION RESEARCH PROGRAM PHASE II ACTION",
"SMALL TECHNOLOGY TRANSFER RESEARCH PROGRAM PHASE I",
"SMALL TECHNOLOGY TRANSFER RESEARCH PROGRAM PHASE II"
) ~ T,
is.na(typeResearch) & isPossibleSBIRSTTR ~ T,
TRUE ~ F
)
)
}
if (all_data %>% hasName("nameFirstInvestigator") &
all_data %>% hasName("nameLastInvestigator")) {
df_pis <-
all_data %>%
distinct(nameFirstInvestigator, nameLastInvestigator) %>%
filter(!is.na(nameFirstInvestigator)) %>%
filter(!nameFirstInvestigator %>% str_detect("UNKNOWN")) %>%
filter(!is.na(nameLastInvestigator) %>% str_detect("UNKNOWN"))
if (nrow(df_pis) > 0) {
df_pis <-
df_pis %>%
mutate(
namePrincipalInvestigator = case_when(
nameFirstInvestigator %>% str_detect("UNKNOWN") ~ NA_character_,
TRUE ~ str_c(nameFirstInvestigator, nameLastInvestigator, sep = " ")
)
) %>%
filter(!is.na(namePrincipalInvestigator))
df_pis <- df_pis %>%
clean_entity_data(entity_column = "namePrincipalInvestigator") %>%
select(-namePrincipalInvestigator) %>%
rename(namePrincipalInvestigator = namePrincipalInvestigatorClean)
all_data <-
all_data %>%
left_join(df_pis,
by = c("nameFirstInvestigator", "nameLastInvestigator"))
}
}
## ENTITY RESOLUTION ### NEEDS FIXING
if (all_data %>% hasName("cageVendor")) {
df_cage <-
all_data %>%
select(idDUNSParent, cageVendor, amountObligation) %>%
filter(!is.na(cageVendor)) %>%
count(idDUNSParent, cageVendor, wt = amountObligation) %>%
group_by(cageVendor) %>%
filter(n == max(n)) %>%
slice(1) %>%
ungroup() %>%
select(idDUNSParent, cageVendor)
all_data <-
all_data %>%
select(-cageVendor) %>%
left_join(df_cage, by = "idDUNSParent") %>%
distinct() %>%
select(one_of(names(all_data)), everything())
}
if (all_data %>% hasName("typeAward")) {
all_data <- all_data %>%
mutate(
typeProcurementFPDS = case_when(
typeAward %>% str_detect("BPA") ~ "BLANKET PURCHASE AGREEMENT",
TRUE ~ typeAward
),
isGrantType = typeAward %>% str_detect("GRANT")
)
}
if (all_data %>% hasName("yearObligation")) {
all_data <- all_data %>%
rename(yearFiscalObligation = yearObligation)
}
all_data <-
all_data %>%
select(-matches("slugDUNS|dateContractSigned")) %>%
select(one_of(c(
"idContractAnalysis", "idContract", "dateObligation"
)), everything())
name_cols <-
all_data %>% select(matches("nameAgency|nameOffice|nameDepartment")) %>% names()
name_cols %>%
walk(function(x) {
all_data <<- .clean_org_usa(data = all_data,
col = x,
clean_entity_column = clean_entity_column)
})
if (exclude_bloat) {
all_data <- all_data %>%
select(-matches("urlFPDS"))
}
actions <- nrow(all_data) %>% comma(digits = 0)
contracts <-
all_data %>% distinct(idContractAnalysis) %>% nrow() %>% comma(digits = 0)
duns <-
all_data %>% distinct(idDUNS) %>% nrow() %>% comma(digits = 0)
parent_duns <-
all_data %>% distinct(idDUNSParent) %>% nrow() %>% comma(digits = 0)
amt <-
all_data$amountObligation %>% sum() %>% currency(digits = 0)
from_date <- all_data$dateObligation %>% min(na.rm = T)
to_date <- all_data$dateObligation %>% max(na.rm = T)
glue(
"\n\n{green({amt})} procured between {red({from_date})} and {red({to_date})} across {yellow(actions)} actions amongst {magenta(contracts)} contracts allocated to {cyan(duns)} distinct DUNS and {blue(parent_duns)} distinct parent DUNS\n\n"
) %>% message()
all_data <- arrange(.data = all_data, dateObligation)
if (snake_names) {
all_data <-
all_data %>%
clean_names()
}
all_data
}
# generate ----------------------------------------------------------------
.generate_atom_urls <-
function(url = "https://fpds.gov/ezsearch/FEEDS/ATOM?s=FPDS&FEEDNAME=PUBLIC&VERSION=1.5.1&q=VENDOR_DUNS_NUMBER:036593457") {
doc <- read_xml(url)
xml_nodes <-
xml_contents(x = doc)
urls <- xml_nodes %>% html_attr("href")
items <- xml_nodes %>% html_attr("rel")
df_url <-
tibble(item = items, url = urls) %>% filter(item == "last")
if (nrow(df_url) == 0) {
return(tibble(urlFPDSAtom = url))
}
total_records <-
df_url$url %>% str_split("\\&start=") %>% flatten_chr() %>% .[[2]] %>% as.numeric()
times <- total_records %/% 10 + 1
seq_ids <- seq(0, by = 10, length.out = times)
urls <-
glue::glue("{url}&start={seq_ids}&sortBy=SIGNED_DATE") %>% as.character() %>% map_chr(URLencode)
data <- tibble(idSequence = 1:times, urlFPDSAtom = urls)
data
}
.generate_base_api_call <-
function(vendor_doing_business_as_name = NULL,
global_vendor_name = NULL,
award_type = NULL,
vendor_name = NULL,
solicitation_procedure = NULL,
vendor_duns_number = NULL,
agency_name = NULL,
contracting_office_name = NULL,
contracting_agency_name = NULL,
principal_naics_code = NULL,
award_status = NULL,
contract_type = NULL,
contract_type_description = NULL,
type_of_contract_pricing = NULL,
contract_id = NULL,
contracting_agency_id = NULL,
contracting_office_id = NULL,
funding_agency_id = NULL,
funding_office_id = NULL,
funding_office_name = NULL,
agency_code = NULL,
department_id = NULL,
subcontract_plan = NULL,
department_name = NULL,
last_mod_date = NULL,
last_modified_by = NULL,
award_completion_date = NULL,
created_date = NULL,
signed_date = NULL,
effective_date = NULL,
estimated_completion_date = NULL,
ref_idv_contract_id = NULL,
cancellation_date = NULL,
destroy_date = NULL,
final_invoice_paid_date = NULL,
funded_through_date = NULL,
last_modified_date = NULL,
physical_completion_date = NULL,
reveal_date = NULL,
solicitation_issue_date = NULL,
sys_last_modified_date = NULL,
vendor_registration_date = NULL,
vendor_renewal_date = NULL,
number_of_offers_received = NULL,
base_exercised_options_value = NULL,
current_contract_value = NULL,
dollars_obligated = NULL,
contract_value = NULL,
fee_range_lower_value = NULL,
fee_range_upper_value = NULL,
fixed_fee_value = NULL,
obligated_amount = NULL,
total_current_contract_value = NULL,
total_dollars_obligated = NULL,
total_non_government_value = NULL,
total_ultimate_contract_value = NULL,
ultimate_contract_value = NULL,
contract_fiscal_year = NULL,
description_of_requirement = NULL,
reason_for_modification = NULL,
legislative_mandates = NULL,
local_area_set_aside = NULL,
socio_economic_indicators = NULL,
multiyear_contract = NULL,
national_interest_code = NULL,
product_or_service_code = NULL,
performance_district_code = NULL,
performance_country = NULL,
performance_state_name = NULL,
vendor_address_city = NULL,
vendor_congress_district_code = NULL,
vendor_address_country_code = NULL,
vendor_address_country_name = NULL,
vendor_address_state_code = NULL,
vendor_address_state_name = NULL,
vendor_address_zip_code = NULL,
extent_competed = NULL,
research = NULL,
...) {
options(scipen = 999999)
params = list(
piid = contract_id,
global_vendor_name = global_vendor_name,
award_type = award_type,
solicitation_proc_description = solicitation_procedure,
subcontract_plan = subcontract_plan,
contracting_office_id = contracting_office_id,
contract_type_description = contract_type_description,
contracting_office_name = contracting_office_name,
funding_agency_id = funding_agency_id,
funding_office_id = funding_office_id,
funding_office_name = funding_office_name,
agency_code = agency_code,
agency_name = agency_name,
department_id = department_id,
department_name = department_name,
last_mod_date = last_mod_date,
last_modified_by = last_modified_by,
award_completion_date = award_completion_date,
cancellation_date = cancellation_date,
destroy_date = destroy_date,
final_invoice_paid_date = final_invoice_paid_date,
funded_through_date = funded_through_date,
last_modified_date = last_modified_date,
physical_completion_date = physical_completion_date,
reveal_date = reveal_date,
solicitation_issue_date = solicitation_issue_date,
sys_last_modified_date = sys_last_modified_date,
vendor_registration_date = vendor_registration_date,
vendor_renewal_date = vendor_renewal_date,
created_date = created_date,
signed_date = signed_date,
effective_date = effective_date,
estimated_completion_date = estimated_completion_date,
number_of_offers_received = number_of_offers_received,
base_exercised_options_value = base_exercised_options_value,
current_contract_value = current_contract_value,
dollars_obligated = dollars_obligated,
contract_value = contract_value,
fee_range_lower_value = fee_range_lower_value,
fee_range_upper_value = fee_range_upper_value,
fixed_fee_value = fixed_fee_value,
obligated_amount = obligated_amount,
total_current_contract_value = total_current_contract_value,
total_dollars_obligated = total_dollars_obligated,
total_non_government_value = total_non_government_value,
total_ultimate_contract_value = total_ultimate_contract_value,
ultimate_contract_value = ultimate_contract_value,
contract_fiscal_year = contract_fiscal_year,
type_of_contract_pricing = type_of_contract_pricing,
award_status = award_status,
contract_type = contract_type,
description_of_requirement = description_of_requirement,
reason_for_modification = reason_for_modification,
legislative_mandates = legislative_mandates,
local_area_set_aside = local_area_set_aside,
socio_economic_indicators = socio_economic_indicators,
multiyear_contract = multiyear_contract,
national_interest_code = national_interest_code,
principal_naics_code = principal_naics_code,
product_or_service_code = product_or_service_code,
pop_congress_district_code = performance_district_code,
pop_country_name = performance_country,
pop_state_name = performance_state_name,
vendor_address_city = vendor_address_city,
vendor_congress_district_code = vendor_congress_district_code,
vendor_address_country_code = vendor_address_country_code,
vendor_address_country_name = vendor_address_country_name,
vendor_duns_number = vendor_duns_number,
vendor_doing_business_as_name = vendor_doing_business_as_name,
vendor_name = vendor_name,
vendor_address_state_code = vendor_address_state_code,
vendor_address_state_name = vendor_address_state_name,
vendor_address_zip_code = vendor_address_zip_code,
research = research,
ref_idv_contract_id = ref_idv_contract_id,
contracting_agency_name = contracting_agency_name,
contracting_office_id = contracting_office_id
)
params <-
utils::modifyList(params, list(...))
params
}
.fpds_api_call_params <-
function(vendor_doing_business_as_name = NULL,
global_vendor_name = NULL,
vendor_name = NULL,
vendor_duns_number = NULL,
award_type = NULL,
solicitation_procedure = NULL,
agency_name = NULL,
contracting_office_name = NULL,
contracting_agency_name = NULL,
principal_naics_code = NULL,
award_status = NULL,
contract_type = NULL,
contract_type_description = NULL,
type_of_contract_pricing = NULL,
contract_id = NULL,
ref_idv_contract_id = NULL,
ref_idv_agency_id = NULL,
modification_number = NULL,
contracting_agency_id = NULL,
contracting_office_id = NULL,
subcontract_plan = NULL,
funding_agency_id = NULL,
funding_office_id = NULL,
funding_office_name = NULL,
agency_code = NULL,
department_id = NULL,
department_name = NULL,
last_mod_date = NULL,
last_modified_by = NULL,
award_completion_date = NULL,
created_date = NULL,
signed_date = NULL,
effective_date = NULL,
estimated_completion_date = NULL,
cancellation_date = NULL,
destroy_date = NULL,
final_invoice_paid_date = NULL,
funded_through_date = NULL,
last_modified_date = NULL,
physical_completion_date = NULL,
reveal_date = NULL,
solicitation_issue_date = NULL,
sys_last_modified_date = NULL,
vendor_registration_date = NULL,
vendor_renewal_date = NULL,
base_exercised_options_value = NULL,
current_contract_value = NULL,
dollars_obligated = NULL,
contract_value = NULL,
fee_range_lower_value = NULL,
fee_range_upper_value = NULL,
fixed_fee_value = NULL,
obligated_amount = NULL,
total_current_contract_value = NULL,
total_dollars_obligated = NULL,
total_non_government_value = NULL,
total_ultimate_contract_value = NULL,
ultimate_contract_value = NULL,
contract_fiscal_year = NULL,
created_by = NULL,
description_of_requirement = NULL,
reason_for_modification = NULL,
legislative_mandates = NULL,
research = NULL,
extent_competed = NULL,
local_area_set_aside = NULL,
socio_economic_indicators = NULL,
multiyear_contract = NULL,
national_interest_code = NULL,
national_interest_description = NULL,
naics_description = NULL,
product_or_service_code = NULL,
product_or_service_description = NULL,
performance_district_code = NULL,
performance_country = NULL,
performance_state_name = NULL,
vendor_address_city = NULL,
vendor_congress_district_code = NULL,
vendor_address_country_code = NULL,
vendor_address_country_name = NULL,
vendor_address_state_code = NULL,
vendor_address_state_name = NULL,
vendor_address_zip_code = NULL,
number_of_offers_received = NULL,
return_query = F,
...) {
result_list <-
.generate_base_api_call(
piid = contract_id,
global_vendor_name = global_vendor_name,
award_type = award_type,
modification_number = modification_number,
contracting_agency_id = contracting_agency_id,
contracting_agency_name = contracting_agency_name,
contracting_office_id = contracting_office_id,
contract_type_description = contract_type_description,
contracting_office_name = contracting_office_name,
funding_agency_id = funding_agency_id,
funding_office_id = funding_office_id,
funding_office_name = funding_office_name,
agency_code = agency_code,
agency_name = agency_name,
department_id = department_id,
department_name = department_name,
last_mod_date = last_mod_date,
last_modified_by = last_modified_by,
award_completion_date = award_completion_date,
subcontract_plan = subcontract_plan,
created_date = created_date,
signed_date = signed_date,
effective_date = effective_date,
estimated_completion_date = estimated_completion_date,
number_of_offers_received = number_of_offers_received,
base_exercised_options_value = base_exercised_options_value,
current_contract_value = current_contract_value,
dollars_obligated = dollars_obligated,
contract_value = contract_value,
fee_range_lower_value = fee_range_lower_value,
fee_range_upper_value = fee_range_upper_value,
fixed_fee_value = fixed_fee_value,
obligated_amount = obligated_amount,
total_current_contract_value = total_current_contract_value,
total_dollars_obligated = total_dollars_obligated,
total_non_government_value = total_non_government_value,
total_ultimate_contract_value = total_ultimate_contract_value,
ultimate_contract_value = ultimate_contract_value,
contract_fiscal_year = contract_fiscal_year,
type_of_contract_pricing = type_of_contract_pricing,
award_status = award_status,
contract_type = contract_type,
description_of_requirement = description_of_requirement,
reason_for_modification = reason_for_modification,
legislative_mandates = legislative_mandates,
local_area_set_aside = local_area_set_aside,
socio_economic_indicators = socio_economic_indicators,
multiyear_contract = multiyear_contract,
national_interest_code = national_interest_code,
national_interest_description = national_interest_description,
principal_naics_code = principal_naics_code,
naics_description = naics_description,
product_or_service_code = product_or_service_code,
product_or_service_description = product_or_service_description,
pop_congress_district_code = performance_district_code,
pop_country_name = performance_country,
pop_state_name = performance_state_name,
vendor_address_city = vendor_address_city,
vendor_congress_district_code = vendor_congress_district_code,
vendor_address_country_code = vendor_address_country_code,
vendor_address_country_name = vendor_address_country_name,
vendor_duns_number = vendor_duns_number,
vendor_doing_business_as_name = vendor_doing_business_as_name,
vendor_name = vendor_name,
vendor_address_state_code = vendor_address_state_code,
vendor_address_state_name = vendor_address_state_name,
vendor_address_zip_code = vendor_address_zip_code,
contract_id = contract_id,
performance_district_code = performance_district_code,
performance_country = performance_country,
performance_state_name = performance_state_name,
extent_competed = extent_competed,
research = research,
cancellation_date = cancellation_date,
destroy_date = destroy_date,
final_invoice_paid_date = final_invoice_paid_date,
funded_through_date = funded_through_date,
last_modified_date = last_modified_date,
physical_completion_date = physical_completion_date,
reveal_date = reveal_date,
solicitation_issue_date = solicitation_issue_date,
sys_last_modified_date = sys_last_modified_date,
vendor_registration_date = vendor_registration_date,
vendor_renewal_date = vendor_renewal_date,
solicitation_procedure = solicitation_procedure,
ref_idv_contract_id = ref_idv_contract_id,
...
)
if (result_list %>% map_dbl(length) %>% sum() == 0) {
stop("No parameters")
}
df_parameters <-
result_list %>% .munge_double_list() %>%
.create_fpds_query_params()
if (return_query) {
df_values <-
df_parameters %>%
gather(param, value)
query <-
df_values %>%
mutate(
value = value %>% str_to_upper() %>% map_chr(URLencode),
param = str_to_upper(param)
) %>%
unite(slug, param, value, sep = ":") %>%
pull(slug) %>%
str_c(collapse = "%20") %>%
map_chr(URLencode)
return(query)
}
df_parameters
}
.generate_fpds_advanced_query <-
function(vendor_doing_business_as_name = NULL,
award_type = NULL,
global_vendor_name = NULL,
vendor_name = NULL,
vendor_duns_number = NULL,
agency_name = NULL,
contracting_office_name = NULL,
solicitation_procedure = NULL,
contracting_agency_name = NULL,
principal_naics_code = NULL,
award_status = NULL,
contract_type = NULL,
contract_type_description = NULL,
type_of_contract_pricing = NULL,
contract_id = NULL,
ref_idv_contract_id = NULL,
ref_idv_agency_id = NULL,
modification_number = NULL,
contracting_agency_id = NULL,
contracting_office_id = NULL,
funding_agency_id = NULL,
funding_office_id = NULL,
funding_office_name = NULL,
subcontract_plan = NULL,
agency_code = NULL,
department_id = NULL,
department_name = NULL,
last_mod_date = NULL,
last_modified_by = NULL,
award_completion_date = NULL,
created_date = NULL,
signed_date = NULL,
effective_date = NULL,
estimated_completion_date = NULL,
cancellation_date = NULL,
destroy_date = NULL,
final_invoice_paid_date = NULL,
funded_through_date = NULL,
last_modified_date = NULL,
physical_completion_date = NULL,
reveal_date = NULL,
solicitation_issue_date = NULL,
sys_last_modified_date = NULL,
vendor_registration_date = NULL,
vendor_renewal_date = NULL,
base_exercised_options_value = NULL,
current_contract_value = NULL,
dollars_obligated = NULL,
contract_value = NULL,
fee_range_lower_value = NULL,
fee_range_upper_value = NULL,
fixed_fee_value = NULL,
obligated_amount = NULL,
total_current_contract_value = NULL,
total_dollars_obligated = NULL,
total_non_government_value = NULL,
total_ultimate_contract_value = NULL,
ultimate_contract_value = NULL,
contract_fiscal_year = NULL,
created_by = NULL,
description_of_requirement = NULL,
reason_for_modification = NULL,
legislative_mandates = NULL,
local_area_set_aside = NULL,
socio_economic_indicators = NULL,
multiyear_contract = NULL,
national_interest_code = NULL,
national_interest_description = NULL,
naics_description = NULL,
product_or_service_code = NULL,
product_or_service_description = NULL,
performance_district_code = NULL,
performance_country = NULL,
performance_state_name = NULL,
research = NULL,
vendor_address_city = NULL,
vendor_congress_district_code = NULL,
vendor_address_country_code = NULL,
vendor_address_country_name = NULL,
vendor_address_state_code = NULL,
vendor_address_state_name = NULL,
vendor_address_zip_code = NULL,
number_of_offers_received = NULL,
extent_competed = NULL,
sort_item = "Signed Date",
sort_descending = T,
return_message = T,
...) {
base_url <-
"https://www.fpds.gov/ezsearch/FEEDS/ATOM?FEEDNAME=PUBLIC&q="
df_parameters <-
.fpds_api_call_params(
contract_id = contract_id,
global_vendor_name = global_vendor_name,
award_type = award_type,
contracting_agency_id = contracting_agency_id,
contracting_agency_name = contracting_agency_name,
contracting_office_id = contracting_office_id,
contract_type_description = contract_type_description,
contracting_office_name = contracting_office_name,
funding_agency_id = funding_agency_id,
funding_office_id = funding_office_id,
funding_office_name = funding_office_name,
subcontract_plan = subcontract_plan,
agency_code = agency_code,
agency_name = agency_name,
department_id = department_id,
department_name = department_name,
last_mod_date = last_mod_date,
last_modified_by = last_modified_by,
award_completion_date = award_completion_date,
created_date = created_date,
signed_date = signed_date,
effective_date = effective_date,
estimated_completion_date = estimated_completion_date,
number_of_offers_received = number_of_offers_received,
base_exercised_options_value = base_exercised_options_value,
current_contract_value = current_contract_value,
dollars_obligated = dollars_obligated,
contract_value = contract_value,
fee_range_lower_value = fee_range_lower_value,
fee_range_upper_value = fee_range_upper_value,
fixed_fee_value = fixed_fee_value,
obligated_amount = obligated_amount,
total_current_contract_value = total_current_contract_value,
total_dollars_obligated = total_dollars_obligated,
total_non_government_value = total_non_government_value,
total_ultimate_contract_value = total_ultimate_contract_value,
ultimate_contract_value = ultimate_contract_value,
contract_fiscal_year = contract_fiscal_year,
type_of_contract_pricing = type_of_contract_pricing,
award_status = award_status,
contract_type = contract_type,
description_of_requirement = description_of_requirement,
reason_for_modification = reason_for_modification,
legislative_mandates = legislative_mandates,
local_area_set_aside = local_area_set_aside,
socio_economic_indicators = socio_economic_indicators,
multiyear_contract = multiyear_contract,
national_interest_code = national_interest_code,
national_interest_description = national_interest_description,
principal_naics_code = principal_naics_code,
naics_description = naics_description,
product_or_service_code = product_or_service_code,
product_or_service_description = product_or_service_description,
performance_district_code = performance_district_code,
performance_country = performance_country,
pop_state_name = performance_state_name,
vendor_address_city = vendor_address_city,
vendor_congress_district_code = vendor_congress_district_code,
vendor_address_country_code = vendor_address_country_code,
vendor_address_country_name = vendor_address_country_name,
vendor_duns_number = vendor_duns_number,
vendor_doing_business_as_name = vendor_doing_business_as_name,
vendor_name = vendor_name,
vendor_address_state_code = vendor_address_state_code,
vendor_address_state_name = vendor_address_state_name,
vendor_address_zip_code = vendor_address_zip_code,
ref_idv_contract_id = ref_idv_contract_id,
ref_idv_agency_id = ref_idv_agency_id,
modification_number = modification_number,
created_by = created_by,
research = research,
extent_competed = extent_competed,
performance_state_name = performance_state_name,
cancellation_date = cancellation_date,
destroy_date = destroy_date,
final_invoice_paid_date = final_invoice_paid_date,
funded_through_date = funded_through_date,
last_modified_date = last_modified_date,
physical_completion_date = physical_completion_date,
reveal_date = reveal_date,
solicitation_issue_date = solicitation_issue_date,
sys_last_modified_date = sys_last_modified_date,
vendor_registration_date = vendor_registration_date,
vendor_renewal_date = vendor_renewal_date,
solicitation_procedure = solicitation_procedure,
return_query = F,
...
) %>%
mutate_all(str_to_upper)
sort_slug <-
.sort_slug(sort_method = sort_item, sort_descending = sort_descending)
df_values <-
df_parameters %>%
gather(param, value)
query <-
df_values %>%
mutate(
value = value %>% str_to_upper() %>% map_chr(URLencode),
param = str_to_upper(param)
) %>%
unite(slug, param, value, sep = ":") %>%
pull(slug) %>%
str_c(collapse = "%20") %>%
map_chr(URLencode)
url <-
glue::glue("{base_url}{query}&{sort_slug}") %>% as.character()
data <-
df_parameters %>%
.munge_fpds_names()
names(data) <-
names(data) %>% str_c("Search")
data <-
data %>%
mutate(urlBase = url)
df_urls <-
.generate_atom_urls(url = url) %>%
mutate(urlBase = url)
df_urls %>%
left_join(data, by = "urlBase") %>%
select(-urlBase)
}
.generate_fpds_advanced_queries <-
function(vendor_doing_business_as_name = NA,
award_type = NA,
global_vendor_name = NA,
vendor_name = NA,
parent_vendor_name = NA,
vendor_duns_number = NA,
parent_duns_number = NA,
agency_name = NA,
contracting_office_name = NA,
contracting_agency_name = NA,
principal_naics_code = NA,
award_status = NA,
solicitation_procedure = NA,
contract_type = NA,
contract_type_description = NA,
type_of_contract_pricing = NA,
contract_id = NA,
subcontract_plan = NA,
ref_idv_contract_id = NA,
ref_idv_agency_id = NA,
contracting_agency_id = NA,
contracting_office_id = NA,
funding_agency_id = NA,
funding_office_id = NA,
funding_office_name = NA,
agency_code = NA,
department_id = NA,
department_name = NA,
research = NA,
last_mod_date = NA,
last_modified_by = NA,
award_completion_date = NA,
created_date = NA,
signed_date = NA,
effective_date = NA,
estimated_completion_date = NA,
cancellation_date = NA,
destroy_date = NA,
final_invoice_paid_date = NA,
funded_through_date = NA,
last_modified_date = NA,
physical_completion_date = NA,
reveal_date = NA,
solicitation_issue_date = NA,
sys_last_modified_date = NA,
vendor_registration_date = NA,
vendor_renewal_date = NA,
base_exercised_options_value = NA,
current_contract_value = NA,
dollars_obligated = NA,
contract_value = NA,
fee_range_lower_value = NA,
fee_range_upper_value = NA,
fixed_fee_value = NA,
obligated_amount = NA,
total_current_contract_value = NA,
total_dollars_obligated = NA,
total_non_government_value = NA,
total_ultimate_contract_value = NA,
ultimate_contract_value = NA,
contract_fiscal_year = NA,
created_by = NA,
description_of_requirement = NA,
reason_for_modification = NA,
legislative_mandates = NA,
local_area_set_aside = NA,
socio_economic_indicators = NA,
multiyear_contract = NA,
national_interest_code = NA,
product_or_service_code = NA,
performance_district_code = NA,
performance_country = NA,
performance_state_name = NA,
vendor_address_city = NA,
vendor_congress_district_code = NA,
vendor_address_country_code = NA,
vendor_address_country_name = NA,
vendor_address_state_code = NA,
vendor_address_state_name = NA,
vendor_address_zip_code = NA,
extent_competed = NA,
number_of_offers_received = NA,
sort_item = "Signed Date",
sort_descending = T,
return_message = T,
...) {
options(warn = -1)
if (!is.na(contract_id) %>% sum(na.rm = T) > 0) {
contract_id <-
contract_id %>% str_remove_all("\\-")
}
if (!is.na(last_mod_date) %>% sum(na.rm = T) > 0 &
length(last_mod_date) == 2) {
last_mod_date <- .shorten_dates(dates = last_mod_date)
}
if (!is.na(award_completion_date) %>% sum(na.rm = T) > 0 &
length(award_completion_date) == 2) {
award_completion_date <-
.shorten_dates(dates = award_completion_date)
}
if (!is.na(created_date) %>% sum(na.rm = T) > 0 &
length(created_date) == 2) {
created_date <- .shorten_dates(dates = created_date)
}
if (!is.na(signed_date) %>% sum(na.rm = T) > 0 &
length(signed_date) == 2) {
signed_date <- .shorten_dates(dates = signed_date)
}
if (!is.na(effective_date) %>% sum(na.rm = T) > 0 &
length(effective_date) == 2) {
effective_date <- .shorten_dates(dates = effective_date)
}
if (!is.na(estimated_completion_date) %>% sum(na.rm = T) > 0 &
length(estimated_completion_date) == 2) {
estimated_completion_date <-
.shorten_dates(dates = estimated_completion_date)
}
if (!is.na(cancellation_date) %>% sum(na.rm = T) > 0 &
length(cancellation_date) == 2) {
cancellation_date <- .shorten_dates(dates = cancellation_date)
}
if (!is.na(destroy_date) %>% sum(na.rm = T) > 0 &
length(destroy_date) == 2) {
destroy_date <- .shorten_dates(dates = destroy_date)
}
if (!is.na(final_invoice_paid_date) %>% sum(na.rm = T) > 0 &
length(final_invoice_paid_date) == 2) {
final_invoice_paid_date <-
.shorten_dates(dates = final_invoice_paid_date)
}
if (!is.na(funded_through_date) %>% sum(na.rm = T) > 0 &
length(funded_through_date) == 2) {
funded_through_date <- .shorten_dates(dates = funded_through_date)
}
if (!is.na(last_modified_date) %>% sum(na.rm = T) > 0 &
length(last_modified_date) == 2) {
last_modified_date <- .shorten_dates(dates = last_modified_date)
}
if (!is.na(physical_completion_date) %>% sum(na.rm = T) > 0 &
length(physical_completion_date) == 2) {
physical_completion_date <-
.shorten_dates(dates = physical_completion_date)
}
if (!is.na(vendor_duns_number)) {
vendor_duns_number <-
.pad_duns(duns = vendor_duns_number, zero_base = 9) %>% pull(slugDUNS)
}
if (!is.na(parent_duns_number)) {
parent_duns_number <-
.pad_duns(duns = parent_duns_number, zero_base = 9) %>% pull(slugDUNS)
}
if (!is.na(reveal_date) %>% sum(na.rm = T) > 0 &
length(reveal_date) == 2) {
reveal_date <- .shorten_dates(dates = reveal_date)
}
if (!is.na(solicitation_issue_date) %>% sum(na.rm = T) > 0 &
length(solicitation_issue_date) == 2) {
solicitation_issue_date <-
.shorten_dates(dates = solicitation_issue_date)
}
if (!is.na(sys_last_modified_date) %>% sum(na.rm = T) > 0 &
length(sys_last_modified_date) == 2) {
sys_last_modified_date <-
.shorten_dates(dates = sys_last_modified_date)
}
if (!is.na(vendor_registration_date) %>% sum(na.rm = T) > 0 &
length(vendor_registration_date) == 2) {
vendor_registration_date <-
.shorten_dates(dates = vendor_registration_date)
}
if (!is.na(vendor_renewal_date) %>% sum(na.rm = T) > 0 &
length(signed_date) == 2) {
vendor_renewal_date <-
.shorten_dates(dates = vendor_renewal_date)
}
if (!is.na(base_exercised_options_value) %>% sum(na.rm = T) > 0 &
length(base_exercised_options_value) == 2) {
base_exercised_options_value <-
.shorten_amounts(base_exercised_options_value)
}
if (!is.na(current_contract_value) %>% sum(na.rm = T) > 0 &
length(current_contract_value) == 2) {
current_contract_value <- .shorten_amounts(current_contract_value)
}
if (!is.na(dollars_obligated) %>% sum(na.rm = T) > 0 &
length(dollars_obligated) == 2) {
dollars_obligated <- .shorten_amounts(dollars_obligated)
}
if (!is.na(contract_value) %>% sum(na.rm = T) > 0 &
length(contract_value) == 2) {
contract_value <- .shorten_amounts(contract_value)
}
if (!is.na(fee_range_lower_value) %>% sum(na.rm = T) > 0 &
length(fee_range_lower_value) == 2) {
fee_range_lower_value <- .shorten_amounts(fee_range_lower_value)
}
if (!is.na(fee_range_upper_value) %>% sum(na.rm = T) > 0 &
length(fee_range_upper_value) == 2) {
fee_range_upper_value <- .shorten_amounts(fee_range_upper_value)
}
if (!is.na(fixed_fee_value) %>% sum(na.rm = T) > 0 &
length(fixed_fee_value) == 2) {
fixed_fee_value <- .shorten_amounts(fixed_fee_value)
}
if (!is.na(obligated_amount) %>% sum(na.rm = T) > 0 &
length(obligated_amount) == 2) {
obligated_amount <- .shorten_amounts(obligated_amount)
}
if (!is.na(total_current_contract_value) %>% sum(na.rm = T) > 0 &
length(total_current_contract_value) == 2) {
total_current_contract_value <-
.shorten_amounts(total_current_contract_value)
}
if (!is.na(total_dollars_obligated) %>% sum(na.rm = T) > 0 &
length(total_dollars_obligated) == 2) {
total_dollars_obligated <- .shorten_amounts(total_dollars_obligated)
}
if (!is.na(total_non_government_value) %>% sum(na.rm = T) > 0 &
length(total_non_government_value) == 2) {
total_non_government_value <-
.shorten_amounts(total_non_government_value)
}
if (!is.na(total_ultimate_contract_value) %>% sum(na.rm = T) > 0 &
length(total_ultimate_contract_value) == 2) {
total_ultimate_contract_value <-
.shorten_amounts(total_ultimate_contract_value)
}
if (!is.na(ultimate_contract_value) %>% sum(na.rm = T) > 0 &
length(ultimate_contract_value) == 2) {
ultimate_contract_value <-
.shorten_amounts(ultimate_contract_value)
}
df_inputs <-
expand.grid(
vendor_doing_business_as_name = vendor_doing_business_as_name,
global_vendor_name = global_vendor_name,
vendor_name = vendor_name,
vendor_duns_number = vendor_duns_number,
agency_name = agency_name,
award_type = award_type,
contracting_office_name = contracting_office_name,
contracting_agency_name = contracting_agency_name,
principal_naics_code = principal_naics_code,
parent_duns_number = parent_duns_number,
parent_vendor_name = parent_vendor_name,
award_status = award_status,
contract_type = contract_type,
contract_type_description = contract_type_description,
type_of_contract_pricing = type_of_contract_pricing,
contract_id = contract_id,
ref_idv_contract_id = ref_idv_contract_id,
ref_idv_agency_id = ref_idv_agency_id ,
contracting_agency_id = contracting_agency_id,
contracting_office_id = contracting_office_id,
funding_agency_id = funding_agency_id,
funding_office_id = funding_office_id,
subcontract_plan = subcontract_plan,
funding_office_name = funding_office_name ,
agency_code = agency_code,
department_id = department_id,
department_name = department_name,
last_mod_date = last_mod_date,
last_modified_by = last_modified_by,
award_completion_date = award_completion_date,
created_date = created_date,
signed_date = signed_date ,
effective_date = effective_date,
estimated_completion_date = estimated_completion_date,
cancellation_date = cancellation_date,
destroy_date = destroy_date,
final_invoice_paid_date = final_invoice_paid_date,
funded_through_date = funded_through_date,
last_modified_date = last_modified_date,
physical_completion_date = physical_completion_date,
reveal_date = reveal_date,
solicitation_issue_date = solicitation_issue_date,
sys_last_modified_date = sys_last_modified_date,
vendor_registration_date = vendor_registration_date,
vendor_renewal_date = vendor_renewal_date,
base_exercised_options_value = base_exercised_options_value,
current_contract_value = current_contract_value,
dollars_obligated = dollars_obligated,
contract_value = contract_value,
fee_range_lower_value = fee_range_lower_value,
fee_range_upper_value = fee_range_upper_value,
fixed_fee_value = fixed_fee_value,
obligated_amount = obligated_amount,
total_current_contract_value = total_current_contract_value,
total_dollars_obligated = total_dollars_obligated,
total_non_government_value = total_non_government_value,
total_ultimate_contract_value = total_ultimate_contract_value,
ultimate_contract_value = ultimate_contract_value,
contract_fiscal_year = contract_fiscal_year,
created_by = created_by,
description_of_requirement = description_of_requirement,
reason_for_modification = reason_for_modification ,
legislative_mandates = legislative_mandates,
local_area_set_aside = local_area_set_aside ,
socio_economic_indicators = socio_economic_indicators ,
multiyear_contract = multiyear_contract,
national_interest_code = national_interest_code,
product_or_service_code = product_or_service_code,
performance_district_code = performance_district_code,
performance_country = performance_country ,
performance_state_name = performance_state_name,
vendor_address_city = vendor_address_city,
vendor_congress_district_code = vendor_congress_district_code,
vendor_address_country_code = vendor_address_country_code,
vendor_address_country_name = vendor_address_country_name,
vendor_address_state_code = vendor_address_state_code,
vendor_address_state_name = vendor_address_state_name,
vendor_address_zip_code = vendor_address_zip_code,
research = research,
solicitation_procedure = solicitation_procedure,
extent_competed = extent_competed,
number_of_offers_received = number_of_offers_received,
stringsAsFactors = F,
...
) %>%
as_tibble()
if (df_inputs %>% gather(item, value) %>% filter(!is.na(value)) %>% nrow() == 0) {
stop("Please enter FPDS search parameter")
}
data <-
1:nrow(df_inputs) %>%
map_dfr(function(x) {
df_row <-
df_inputs %>% dplyr::slice(x) %>% as_tibble() %>% mutate_if(is.factor, as.character)
if (is.na(df_row$vendor_doing_business_as_name)) {
vendor_doing_business_as_name_slug <- NULL
} else {
vendor_doing_business_as_name_slug <-
df_row$vendor_doing_business_as_name
}
if (is.na(df_row$global_vendor_name)) {
global_vendor_name_slug <- NULL
} else {
global_vendor_name_slug <- df_row$global_vendor_name
}
if (is.na(df_row$subcontract_plan)) {
subcontract_plan_slug <- NULL
} else {
subcontract_plan_slug <- df_row$subcontract_plan
}
if (is.na(df_row$solicitation_procedure)) {
solicitation_procedure_slug <- NULL
} else {
solicitation_procedure_slug <- df_row$solicitation_procedure
}
if (is.na(df_row$research)) {
research_slug <- NULL
} else {
research_slug <- df_row$research
}
if (is.na(df_row$award_type)) {
award_type_slug <- NULL
} else {
award_type_slug <- df_row$award_type
}
if (is.na(df_row$vendor_name)) {
vendor_name_slug <- NULL
} else {
vendor_name_slug <- df_row$vendor_name
}
if (is.na(df_row$parent_vendor_name)) {
parent_vendor_name_slug <- NULL
} else {
parent_vendor_name_slug <- df_row$parent_vendor_name
}
if (is.na(df_row$vendor_duns_number)) {
vendor_duns_number_slug <- NULL
} else {
vendor_duns_number_slug <- df_row$vendor_duns_number
}
if (is.na(df_row$parent_duns_number)) {
parent_duns_number_slug <- NULL
} else {
parent_duns_number_slug <- df_row$parent_duns_number
}
if (is.na(df_row$agency_name)) {
agency_name_slug <- NULL
} else {
agency_name_slug <- df_row$agency_name
}
if (is.na(df_row$contracting_office_name)) {
contracting_office_name_slug <- NULL
} else {
contracting_office_name_slug <- df_row$contracting_office_name
}
if (is.na(df_row$contracting_agency_name)) {
contracting_agency_name_slug <- NULL
} else {
contracting_agency_name_slug <- df_row$contracting_agency_name
}
if (is.na(df_row$principal_naics_code)) {
principal_naics_code_slug <- NULL
} else {
principal_naics_code_slug <- df_row$principal_naics_code
}
if (is.na(df_row$award_status)) {
award_status_slug <- NULL
} else {
award_status_slug <- df_row$award_status
}
if (is.na(df_row$contract_type)) {
contract_type_slug <- NULL
} else {
contract_type_slug <- df_row$contract_type
}
if (is.na(df_row$contract_type_description)) {
contract_type_description_slug <- NULL
} else {
contract_type_description_slug <- df_row$contract_type_description
}
if (is.na(df_row$type_of_contract_pricing)) {
type_of_contract_pricing_slug <- NULL
} else {
type_of_contract_pricing_slug <- df_row$type_of_contract_pricing
}
if (is.na(df_row$contract_id)) {
contract_id_slug <- NULL
} else {
contract_id_slug <- df_row$contract_id
}
if (is.na(df_row$ref_idv_contract_id)) {
ref_idv_contract_id_slug <- NULL
} else {
ref_idv_contract_id_slug <- df_row$ref_idv_contract_id
}
if (is.na(df_row$ref_idv_agency_id)) {
ref_idv_agency_id_slug <- NULL
} else {
ref_idv_agency_id_slug <- df_row$ref_idv_agency_id
}
if (is.na(df_row$contracting_agency_id)) {
contracting_agency_id_slug <- NULL
} else {
contracting_agency_id_slug <- df_row$contracting_agency_id
}
if (is.na(df_row$contracting_office_id)) {
contracting_office_id_slug <- NULL
} else {
contracting_office_id_slug <- df_row$contracting_office_id
}
if (is.na(df_row$funding_agency_id)) {
funding_agency_id_slug <- NULL
} else {
funding_agency_id_slug <- df_row$funding_agency_id
}
if (is.na(df_row$funding_office_id)) {
funding_office_id_slug <- NULL
} else {
funding_office_id_slug <- df_row$funding_office_id
}
if (is.na(df_row$funding_office_name)) {
funding_office_name_slug <- NULL
} else {
funding_office_name_slug <- df_row$funding_office_name
}
if (is.na(df_row$agency_code)) {
agency_code_slug <- NULL
} else {
agency_code_slug <- df_row$agency_code
}
if (is.na(df_row$department_id)) {
department_id_slug <- NULL
} else {
department_id_slug <- df_row$department_id
}
if (is.na(df_row$department_name)) {
department_name_slug <- NULL
} else {
department_name_slug <- df_row$department_name
}
if (is.na(df_row$last_mod_date)) {
last_mod_date_slug <- NULL
} else {
last_mod_date_slug <- df_row$last_mod_date
}
if (is.na(df_row$last_modified_by)) {
last_modified_by_slug <- NULL
} else {
last_modified_by_slug <- df_row$last_modified_by
}
if (is.na(df_row$award_completion_date)) {
award_completion_date_slug <- NULL
} else {
award_completion_date_slug <- df_row$award_completion_date
}
if (is.na(df_row$created_date)) {
created_date_slug <- NULL
} else {
created_date_slug <- df_row$created_date
}
if (is.na(df_row$signed_date)) {
signed_date_slug <- NULL
} else {
signed_date_slug <- df_row$signed_date
}
if (is.na(df_row$effective_date)) {
effective_date_slug <- NULL
} else {
effective_date_slug <- df_row$effective_date
}
if (is.na(df_row$estimated_completion_date)) {
estimated_completion_date_slug <- NULL
} else {
estimated_completion_date_slug <- df_row$estimated_completion_date
}
if (is.na(df_row$cancellation_date)) {
cancellation_date_slug <- NULL
} else {
cancellation_date_slug <- df_row$cancellation_date
}
if (is.na(df_row$destroy_date)) {
destroy_date_slug <- NULL
} else {
destroy_date_slug <- df_row$destroy_date
}
if (is.na(df_row$final_invoice_paid_date)) {
final_invoice_paid_date_slug <- NULL
} else {
final_invoice_paid_date_slug <- df_row$final_invoice_paid_date
}
if (is.na(df_row$funded_through_date)) {
funded_through_date_slug <- NULL
} else {
funded_through_date_slug <- df_row$funded_through_date
}
if (is.na(df_row$last_modified_date)) {
last_modified_date_slug <- NULL
} else {
last_modified_date_slug <- df_row$last_modified_date
}
if (is.na(df_row$physical_completion_date)) {
physical_completion_date_slug <- NULL
} else {
physical_completion_date_slug <- df_row$physical_completion_date
}
if (is.na(df_row$reveal_date)) {
reveal_date_slug <- NULL
} else {
reveal_date_slug <- df_row$reveal_date
}
if (is.na(df_row$solicitation_issue_date)) {
solicitation_issue_date_slug <- NULL
} else {
solicitation_issue_date_slug <- df_row$solicitation_issue_date
}
if (is.na(df_row$sys_last_modified_date)) {
sys_last_modified_date_slug <- NULL
} else {
sys_last_modified_date_slug <- df_row$sys_last_modified_date
}
if (is.na(df_row$vendor_registration_date)) {
vendor_registration_date_slug <- NULL
} else {
vendor_registration_date_slug <- df_row$vendor_registration_date
}
if (is.na(df_row$vendor_renewal_date)) {
vendor_renewal_date_slug <- NULL
} else {
vendor_renewal_date_slug <-
df_row$vendor_revendor_renewal_dategistration_date
}
if (is.na(df_row$obligated_amount)) {
obligated_amount_slug <- NULL
} else {
obligated_amount_slug <- df_row$obligated_amount
}
if (is.na(df_row$base_exercised_options_value)) {
base_exercised_options_value_slug <- NULL
} else {
base_exercised_options_value_slug <-
df_row$base_exercised_options_value
}
if (is.na(df_row$ultimate_contract_value)) {
ultimate_contract_value_slug <- NULL
} else {
ultimate_contract_value_slug <- df_row$ultimate_contract_value
}
if (is.na(df_row$created_by)) {
created_by_slug <- NULL
} else {
created_by_slug <- df_row$created_by
}
if (is.na(df_row$description_of_requirement)) {
description_of_requirement_slug <- NULL
} else {
description_of_requirement_slug <- df_row$description_of_requirement
}
if (is.na(df_row$legislative_mandates)) {
legislative_mandates_slug <- NULL
} else {
legislative_mandates_slug <- df_row$legislative_mandates
}
if (is.na(df_row$local_area_set_aside)) {
local_area_set_aside_slug <- NULL
} else {
local_area_set_aside_slug <- df_row$local_area_set_aside
}
if (is.na(df_row$socio_economic_indicators)) {
socio_economic_indicators_slug <- NULL
} else {
socio_economic_indicators_slug <- df_row$socio_economic_indicators
}
if (is.na(df_row$multiyear_contract)) {
multiyear_contract_slug <- NULL
} else {
multiyear_contract_slug <- df_row$multiyear_contract
}
if (is.na(df_row$national_interest_code)) {
national_interest_code_slug <- NULL
} else {
national_interest_code_slug <- df_row$national_interest_code
}
if (is.na(df_row$product_or_service_code)) {
product_or_service_code_slug <- NULL
} else {
product_or_service_code_slug <- df_row$product_or_service_code
}
if (is.na(df_row$performance_district_code)) {
performance_district_code_slug <- NULL
} else {
performance_district_code_slug <- df_row$performance_district_code
}
if (is.na(df_row$performance_country)) {
performance_country_slug <- NULL
} else {
performance_country_slug <- df_row$performance_country
}
if (is.na(df_row$performance_state_name)) {
performance_state_name_slug <- NULL
} else {
performance_state_name_slug <- df_row$performance_state_name
}
if (is.na(df_row$vendor_address_city)) {
vendor_address_city_slug <- NULL
} else {
vendor_address_city_slug <- df_row$vendor_address_city
}
if (is.na(df_row$vendor_congress_district_code)) {
vendor_congress_district_code_slug <- NULL
} else {
vendor_congress_district_code_slug <-
df_row$vendor_congress_district_code
}
if (is.na(df_row$vendor_address_country_code)) {
vendor_address_country_code_slug <- NULL
} else {
vendor_address_country_code_slug <-
df_row$vendor_address_country_code
}
if (is.na(vendor_address_country_name)) {
vendor_address_country_name_slug <- NULL
} else {
vendor_address_country_name_slug <-
df_row$vendor_address_country_name
}
if (is.na(df_row$vendor_address_state_code)) {
vendor_address_state_code_slug <- NULL
} else {
vendor_address_state_code_slug <- df_row$vendor_address_state_code
}
if (is.na(df_row$vendor_address_state_name)) {
vendor_address_state_name_slug <- NULL
} else {
vendor_address_state_name_slug <- df_row$vendor_address_state_name
}
if (is.na(df_row$vendor_address_zip_code)) {
vendor_address_zip_code_slug <- NULL
} else {
vendor_address_zip_code_slug <- df_row$vendor_address_zip_code
}
if (is.na(df_row$reason_for_modification)) {
reason_for_modification_slug <- NULL
} else {
reason_for_modification_slug <- df_row$reason_for_modification
}
if (is.na(current_contract_value)) {
current_contract_value_slug <- NULL
} else {
current_contract_value_slug <- df_row$current_contract_value
}
if (is.na(dollars_obligated)) {
dollars_obligated_slug <- NULL
} else {
dollars_obligated_slug <- df_row$dollars_obligated
}
if (is.na(contract_value)) {
contract_value_slug <- NULL
} else {
contract_value_slug <- df_row$contract_value
}
if (is.na(fee_range_lower_value)) {
fee_range_lower_value_slug <- NULL
} else {
fee_range_lower_value_slug <- df_row$fee_range_lower_value
}
if (is.na(fee_range_upper_value)) {
fee_range_upper_value_slug <- NULL
} else {
}
if (is.na(fixed_fee_value)) {
fixed_fee_value_slug <- NULL
} else {
fixed_fee_value_slug <- df_row$fixed_fee_value
}
if (is.na(total_current_contract_value)) {
total_current_contract_value_slug <- NULL
} else {
total_current_contract_value_slug <-
df_row$total_current_contract_value
}
if (is.na(total_dollars_obligated)) {
total_dollars_obligated_slug <- NULL
} else {
total_dollars_obligated_slug <- df_row$total_dollars_obligated
}
if (is.na(total_non_government_value)) {
total_non_government_value_slug <- NULL
} else {
total_non_government_value_slug <- df_row$total_non_government_value
}
if (is.na(total_ultimate_contract_value)) {
total_ultimate_contract_value_slug <- NULL
} else {
total_ultimate_contract_value_slug <-
df_row$total_ultimate_contract_value
}
if (is.na(contract_fiscal_year)) {
contract_fiscal_year_slug <- NULL
} else {
contract_fiscal_year_slug <- df_row$contract_fiscal_year
}
if (is.na(extent_competed)) {
extent_competed_slug <- NULL
} else {
extent_competed_slug <- df_row$extent_competed
}
if (is.na(number_of_offers_received)) {
number_of_offers_received_slug <- NULL
} else {
number_of_offers_received_slug <- df_row$number_of_offers_received
}
.generate_fpds_advanced_query(
vendor_doing_business_as_name = vendor_doing_business_as_name_slug,
global_vendor_name = global_vendor_name_slug,
vendor_name = vendor_name_slug,
parent_vendor_name = parent_vendor_name_slug,
vendor_duns_number = vendor_duns_number_slug,
parent_duns_number = parent_duns_number_slug,
agency_name = agency_name_slug,
contracting_office_name = contracting_office_name_slug,
contracting_agency_name = contracting_agency_name_slug,
principal_naics_code = principal_naics_code_slug,
award_status = award_status_slug,
contract_type = contract_type_slug,
contract_type_description = contract_type_description_slug,
type_of_contract_pricing = type_of_contract_pricing_slug,
contract_id = contract_id_slug,
ref_idv_contract_id = ref_idv_contract_id_slug,
ref_idv_agency_id = ref_idv_agency_id_slug,
contracting_agency_id = contracting_agency_id_slug,
contracting_office_id = contracting_office_id_slug,
funding_agency_id = funding_agency_id_slug,
funding_office_id = funding_office_id_slug,
funding_office_name = funding_office_name_slug,
agency_code = agency_code_slug,
department_id = department_id_slug,
department_name = department_name_slug,
research = research_slug,
last_mod_date = last_mod_date_slug,
last_modified_by = last_modified_by_slug,
award_completion_date = award_completion_date_slug,
created_date = created_date_slug,
signed_date = signed_date_slug,
effective_date = effective_date_slug,
estimated_completion_date = estimated_completion_date_slug,
obligated_amount = obligated_amount_slug,
base_exercised_options_value = base_exercised_options_value_slug,
ultimate_contract_value = ultimate_contract_value_slug,
created_by = created_by_slug,
description_of_requirement = description_of_requirement_slug,
reason_for_modification = reason_for_modification_slug,
legislative_mandates = legislative_mandates_slug,
local_area_set_aside = local_area_set_aside_slug,
socio_economic_indicators = socio_economic_indicators_slug,
multiyear_contract = multiyear_contract_slug,
national_interest_code = national_interest_code_slug,
product_or_service_code = product_or_service_code_slug,
performance_district_code = performance_district_code_slug,
performance_country = performance_country_slug,
performance_state_name = performance_state_name_slug,
vendor_address_city = vendor_address_city_slug,
vendor_congress_district_code = vendor_congress_district_code_slug,
vendor_address_country_code = vendor_address_country_code_slug,
vendor_address_country_name = vendor_address_country_name_slug,
vendor_address_state_code = vendor_address_state_code_slug,
vendor_address_state_name = vendor_address_state_name_slug,
vendor_address_zip_code = vendor_address_zip_code_slug,
sort_item = sort_item,
sort_descending = sort_descending,
return_message = return_message,
cancellation_date = cancellation_date_slug,
destroy_date = destroy_date_slug,
final_invoice_paid_date = final_invoice_paid_date_slug,
funded_through_date = funded_through_date_slug,
last_modified_date = last_modified_date_slug,
physical_completion_date = physical_completion_date_slug,
reveal_date = reveal_date_slug,
solicitation_issue_date = solicitation_issue_date_slug,
sys_last_modified_date = sys_last_modified_date_slug,
vendor_registration_date = vendor_registration_date_slug,
vendor_renewal_date = vendor_renewal_date_slug,
award_type = award_type_slug,
solicitation_procedure = solicitation_procedure_slug,
subcontract_plan = subcontract_plan_slug,
current_contract_value = current_contract_value_slug,
dollars_obligated = dollars_obligated_slug,
contract_value = contract_value_slug,
fee_range_lower_value = fee_range_lower_value_slug,
fee_range_upper_value = fee_range_upper_value_slug,
fixed_fee_value = fixed_fee_value_slug,
total_current_contract_value = total_current_contract_value_slug,
total_dollars_obligated = total_dollars_obligated_slug,
total_non_government_value = total_non_government_value_slug,
total_ultimate_contract_value = total_ultimate_contract_value_slug,
contract_fiscal_year = contract_fiscal_year_slug,
extent_competed = extent_competed_slug,
number_of_offers_received = number_of_offers_received_slug,
...
)
}) %>%
dplyr::select(which(colMeans(is.na(.)) < 1))
data
}
.count_final_atom_page <-
function(url = "https://www.fpds.gov/ezsearch/FEEDS/ATOM?FEEDNAME=PUBLIC&q=%22BRESLER%22&start=10&sortBy=SIGNED_DATE&start=10&sortBy=SIGNED_DATE") {
doc <- read_xml(url)
xml_nodes <-
doc %>%
xml_contents()
tables <-
xml_nodes %>%
xml_name()
df_table_nodes <-
tibble(table = tables) %>%
mutate(idTable = 1:n())
df_ids <-
df_table_nodes %>%
filter(table == "entry")
nrow(df_ids)
}
.generate_fpds_ft_query <-
function(term = "BRESLER",
use_quote = T,
return_message = T) {
base_url <-
"https://www.fpds.gov/ezsearch/FEEDS/ATOM?FEEDNAME=PUBLIC&q="
if (use_quote) {
slug <-
glue::glue('"{term}"') %>% as.character()
} else {
slug <- term
}
query <-
URLencode(slug)
url <- glue::glue("{base_url}{query}") %>% as.character()
df_urls <-
.generate_atom_urls(url = url) %>%
mutate(urlBase = url)
final_url <- df_urls %>%
dplyr::slice(nrow(df_urls)) %>%
pull(urlFPDSAtom)
count_final <- .count_final_atom_page(url = final_url)
total_results <-
(10 * (length(df_urls$urlFPDSAtom) - 1)) + count_final
if (total_results == 0) {
glue("No results for {term}") %>% message()
data <- tibble(
slugSearch = slug,
termSearch = term,
isQuoted = use_quote,
countResults = total_results
)
return()
}
if (return_message) {
slug <- URLdecode(query)
glue("FPDS search of {slug} has {comma(total_results, digits = 0)} results") %>% message()
}
df_urls %>%
mutate(
slugSearch = slug,
termSearch = term,
isQuoted = use_quote,
countResults = total_results
) %>%
select(termSearch, slugSearch, isQuoted, countResults, everything()) %>%
select(-one_of("idSequence", "urlBase"))
}
# atom query -------------------------------------------------------------------
#' FDPS Free Text Search
#'
#' @param terms
#' @param use_quote
#' @param parse_contracts
#' @param return_message
#' @param clean_entity_column
#' @param snake_names
#' @param keep_key_columns
#' @param exclude_bloat
#' @param unformat
#'
#' @return
#' @export
#'
#' @examples
#' fpds_free_text_search(terms = "Palantir Technologies")
fpds_free_text_search <-
function(terms = NULL,
use_quote = T,
use_future = F,
show_progress = T,
parse_contracts = F,
keep_key_columns = F,
clean_entity_column = T,
exclude_bloat = F,
snake_names = F,
clean_address = T,
unformat = F,
return_message = T) {
if (length(terms) == 0) {
stop("Enter search terms")
}
df_terms <-
expand.grid(term = terms,
use_quote = use_quote,
stringsAsFactors = F) %>%
as_tibble()
df_urls <-
1:nrow(df_terms) %>%
map_dfr(function(x) {
df_row <- df_terms %>% dplyr::slice(x)
term <- df_row$term %>% str_to_upper()
quote <- df_row$use_quote
d <- .generate_fpds_ft_query(term = term,
use_quote = quote)
d
})
if (!parse_contracts) {
return(df_urls)
}
urls <- df_urls$urlFPDSAtom
all_data <-
parse_fpds_atom_urls(
urls = urls,
return_message = return_message,
keep_key_columns = keep_key_columns,
exclude_bloat = exclude_bloat,
clean_entity_column = clean_entity_column,
unformat = unformat,
use_future = use_future,
clean_address = clean_address,
snake_names = snake_names
)
all_data$urlFPDSAtom %>% unique()
all_data <-
all_data %>%
mutate(id = urlFPDSAtom %>% str_to_upper()) %>%
left_join(
df_urls %>%
mutate(id = urlFPDSAtom %>% str_to_upper()) %>%
select(termSearch, countResults, countResults, id)
,
by = "id"
) %>%
select(termSearch, countResults, everything()) %>%
select(-id)
group_names <- c(
"idContract",
"nameContract",
"nameAgencyAward",
"dateContractSigned",
"amountObligation",
"numberTransaction",
"idContractIDV",
"nameVendorLegal",
"idDUNSParent"
)
group_cols <- names(all_data)[names(all_data) %in% group_names]
df_counts <- all_data %>%
group_by(!!!syms(group_cols)) %>%
summarise(
termsSearch = unique(termSearch) %>% sort() %>% str_c(collapse = " | "),
countMatchTerms = length(unique(termSearch))
)
all_data <-
all_data %>%
select(-one_of(c(
"termSearch", "countResults", "slugSearch", "isQuoted"
))) %>%
distinct() %>%
left_join(df_counts, by = group_cols) %>%
select(termsSearch, countMatchTerms, everything())
if (snake_names) {
all_data <- all_data %>%
janitor::clean_names()
}
all_data
}
#' FDPS Advanced Search
#'
#' @param vendor_doing_business_as_name
#' @param global_vendor_name
#' @param vendor_name
#' @param vendor_duns_number
#' @param agency_name
#' @param contracting_office_name
#' @param contracting_agency_name
#' @param principal_naics_code
#' @param award_status
#' @param contract_type
#' @param type_of_contract_pricing contract pricing \itemize{
#' \item NULL
#' \item FIXED
#' \item NO FEE
#' \item RANGE - VARIES BY AMOUNT
#' \item RANGE - VARIES BY OTHER FACTOR
#' }
#' @param contract_id contract id
#' @param ref_idv_contract_id
#' @param ref_idv_agency_id
#' @param contracting_agency_id
#' @param contracting_office_id
#' @param funding_agency_id
#' @param funding_office_id
#' @param funding_office_name
#' @param agency_code
#' @param department_id
#' @param department_name
#' @param last_mod_date
#' @param last_modified_by
#' @param award_completion_date
#' @param created_date
#' @param signed_date
#' @param effective_date
#' @param estimated_completion_date
#' @param obligated_amount
#' @param ultimate_contract_value
#' @param created_by
#' @param description_of_requirement
#' @param reason_for_modification
#' @param legislative_mandates
#' @param local_area_set_aside
#' @param socio_economic_indicators
#' @param multiyear_contract
#' @param national_interest_code
#' @param product_or_service_code
#' @param vendor_address_city
#' @param vendor_congress_district_code
#' @param vendor_address_country_code
#' @param vendor_address_country_name
#' @param vendor_address_state_code
#' @param vendor_address_state_name
#' @param vendor_address_zip_code
#' @param parse_contracts
#' @param return_message
#' @param contract_type_description
#' @param parent_vendor_name
#' @param research
#' @param parent_duns_number
#' @param cancellation_date
#' @param destroy_date
#' @param final_invoice_paid_date
#' @param funded_through_date
#' @param last_modified_date
#' @param physical_completion_date
#' @param reveal_date
#' @param solicitation_issue_date
#' @param sys_last_modified_date
#' @param vendor_registration_date
#' @param vendor_renewal_date
#' @param base_exercised_options_value
#' @param performance_district_code
#' @param performance_country
#' @param performance_state_name
#' @param sort_item
#' @param sort_descending
#' @param ...
#' @param award_type
#' @param solicitation_procedure
#' @param subcontract_plan
#' @param current_contract_value
#' @param dollars_obligated
#' @param contract_value
#' @param fee_range_lower_value
#' @param fee_range_upper_value
#' @param fixed_fee_value
#' @param total_current_contract_value
#' @param total_dollars_obligated
#' @param total_non_government_value
#' @param total_ultimate_contract_value
#' @param contract_fiscal_year
#' @param extent_competed
#' @param number_of_offers_received
#' @param clean_entity_column
#' @param snake_names
#' @param keep_key_columns
#' @param unformat
#' @param exclude_bloat
#'
#' @return
#' @export
#'
#' @examples
#'fpds_atom(global_vendor_name = "PALANTIR TECHNOLOGIES", signed_date = c("2019-01-01", "2019-05-02"))
fpds_atom <-
function(global_vendor_name = NA,
vendor_name = NA,
parent_vendor_name = NA,
department_name = NA,
award_type = NA,
research = NA,
vendor_duns_number = NA,
parent_duns_number = NA,
vendor_doing_business_as_name = NA,
agency_name = NA,
contracting_office_name = NA,
contracting_agency_name = NA,
principal_naics_code = NA,
award_status = NA,
subcontract_plan = NA,
solicitation_procedure = NA,
contract_type = NA,
contract_type_description = NA,
type_of_contract_pricing = NA,
contract_id = NA,
ref_idv_contract_id = NA,
ref_idv_agency_id = NA,
contracting_agency_id = NA,
contracting_office_id = NA,
funding_agency_id = NA,
funding_office_id = NA,
funding_office_name = NA,
agency_code = NA,
department_id = NA,
last_mod_date = NA,
last_modified_by = NA,
award_completion_date = NA,
created_date = NA,
signed_date = NA,
effective_date = NA,
estimated_completion_date = NA,
cancellation_date = NA,
destroy_date = NA,
final_invoice_paid_date = NA,
funded_through_date = NA,
last_modified_date = NA,
physical_completion_date = NA,
reveal_date = NA,
solicitation_issue_date = NA,
sys_last_modified_date = NA,
vendor_registration_date = NA,
vendor_renewal_date = NA,
base_exercised_options_value = NA,
current_contract_value = NA,
dollars_obligated = NA,
contract_value = NA,
fee_range_lower_value = NA,
fee_range_upper_value = NA,
fixed_fee_value = NA,
obligated_amount = NA,
total_current_contract_value = NA,
total_dollars_obligated = NA,
total_non_government_value = NA,
total_ultimate_contract_value = NA,
ultimate_contract_value = NA,
contract_fiscal_year = NA,
created_by = NA,
description_of_requirement = NA,
reason_for_modification = NA,
legislative_mandates = NA,
local_area_set_aside = NA,
socio_economic_indicators = NA,
multiyear_contract = NA,
national_interest_code = NA,
product_or_service_code = NA,
performance_district_code = NA,
performance_country = NA,
performance_state_name = NA,
vendor_address_city = NA,
vendor_congress_district_code = NA,
vendor_address_country_code = NA,
vendor_address_country_name = NA,
vendor_address_state_code = NA,
vendor_address_state_name = NA,
vendor_address_zip_code = NA,
extent_competed = NA,
number_of_offers_received = NA,
sort_item = "Signed Date",
use_future = F,
show_progress = T,
clean_address = T,
clean_entity_column = T,
sort_descending = T,
parse_contracts = F,
snake_names = F,
keep_key_columns = F,
exclude_bloat = F,
unformat = T,
return_message = T,
...) {
options(warn = -1)
df_urls <-
.generate_fpds_advanced_queries(
vendor_doing_business_as_name = vendor_doing_business_as_name,
award_type = award_type,
solicitation_procedure = solicitation_procedure,
global_vendor_name = global_vendor_name,
vendor_name = vendor_name,
parent_vendor_name = parent_vendor_name,
vendor_duns_number = vendor_duns_number,
parent_duns_number = parent_duns_number,
agency_name = agency_name,
contracting_office_name = contracting_office_name,
contracting_agency_name = contracting_agency_name,
principal_naics_code = principal_naics_code,
award_status = award_status,
contract_type = contract_type,
contract_type_description = contract_type_description,
type_of_contract_pricing = type_of_contract_pricing,
contract_id = contract_id,
ref_idv_contract_id = ref_idv_contract_id,
ref_idv_agency_id = ref_idv_agency_id,
contracting_agency_id = contracting_agency_id,
contracting_office_id = contracting_office_id,
funding_agency_id = funding_agency_id,
funding_office_id = funding_office_id,
funding_office_name = funding_office_name,
agency_code = agency_code,
department_id = department_id,
department_name = department_name,
research = research,
last_mod_date = last_mod_date,
last_modified_by = last_modified_by,
award_completion_date = award_completion_date,
created_date = created_date,
signed_date = signed_date,
effective_date = effective_date,
estimated_completion_date = estimated_completion_date,
cancellation_date = cancellation_date,
destroy_date = destroy_date,
final_invoice_paid_date = final_invoice_paid_date,
funded_through_date = funded_through_date,
last_modified_date = last_modified_date,
physical_completion_date = physical_completion_date,
reveal_date = reveal_date,
solicitation_issue_date = solicitation_issue_date,
sys_last_modified_date = sys_last_modified_date,
vendor_registration_date = vendor_registration_date,
vendor_renewal_date = vendor_renewal_date,
base_exercised_options_value = base_exercised_options_value,
current_contract_value = current_contract_value,
dollars_obligated = dollars_obligated,
contract_value = contract_value,
fee_range_lower_value = fee_range_lower_value,
fee_range_upper_value = fee_range_upper_value,
fixed_fee_value = fixed_fee_value,
obligated_amount = obligated_amount,
total_current_contract_value = total_current_contract_value,
total_dollars_obligated = total_dollars_obligated,
total_non_government_value = total_non_government_value,
total_ultimate_contract_value = total_ultimate_contract_value,
ultimate_contract_value = ultimate_contract_value,
contract_fiscal_year = contract_fiscal_year,
created_by = created_by,
description_of_requirement = description_of_requirement,
reason_for_modification = reason_for_modification,
legislative_mandates = legislative_mandates,
local_area_set_aside = local_area_set_aside,
socio_economic_indicators = socio_economic_indicators,
multiyear_contract = multiyear_contract,
national_interest_code = national_interest_code,
product_or_service_code = product_or_service_code,
performance_district_code = performance_district_code,
performance_country = performance_country,
performance_state_name = performance_state_name,
vendor_address_city = vendor_address_city,
vendor_congress_district_code = vendor_congress_district_code,
vendor_address_country_code = vendor_address_country_code,
vendor_address_country_name = vendor_address_country_name,
vendor_address_state_code = vendor_address_state_code,
vendor_address_state_name = vendor_address_state_name,
vendor_address_zip_code = vendor_address_zip_code,
sort_item = sort_item,
sort_descending = sort_descending,
return_message = return_message,
subcontract_plan = subcontract_plan,
extent_competed = extent_competed,
number_of_offers_received = number_of_offers_received,
...
)
if (!parse_contracts) {
return(df_urls)
}
urls <- df_urls$urlFPDSAtom
parse_fpds_atom_urls_safe <-
possibly(parse_fpds_atom_urls, tibble())
all_data <-
parse_fpds_atom_urls(
urls = urls,
return_message = return_message,
unformat = unformat,
clean_entity_column = clean_entity_column,
keep_key_columns = keep_key_columns,
exclude_bloat = exclude_bloat,
use_future = use_future,
clean_address = clean_address,
snake_names = F,
)
if (nrow(all_data) == 0) {
"No Matches" %>% message()
return(invisible())
}
all_data <-
all_data %>%
left_join(df_urls, by = "urlFPDSAtom") %>%
.remove_na()
# all_data <-
# all_data %>%
# resolve_listed_duns() %>%
# add_department_codes()
#
if (snake_names) {
all_data <- all_data %>%
janitor::clean_names()
}
all_data
}
#' FPDS Free Text Term Count
#'
#' Searches FPDS for the number of contracts
#' for a vector of specified terms
#'
#' @param terms vector of terms
#' @param use_quote if \code{TRUE} uses quoted word
#' @param return_message if \code{TRUE} returns a message
#' @param clean_entity_column
#' @param snake_names
#'
#' @return
#' @export
#'
#' @examples
#' fpds_free_text_term_count(terms = c("Deep Learning", "Machine Learning"), use_quote = c(T,F))
fpds_free_text_term_count <-
function(terms = NULL,
use_quote = c(T, F),
clean_entity_column = T,
snake_names = F,
return_message = T) {
if (length(terms) == 0) {
stop("No terms entered")
}
data <-
fpds_free_text_search(
terms = terms,
use_quote = use_quote,
parse_contracts = F,
return_message = return_message
)
if (nrow(data) == 0) {
"No matches" %>% message()
}
data <-
data %>%
select(-urlFPDSAtom) %>%
distinct() %>%
arrange(desc(countResults))
if (snake_names) {
data <- data %>% janitor::clean_names()
}
data
}
#' Add Solictiation Groups for FPDS Data
#'
#' @param data a \code{tibble}
#'
#' @return
#' @export
#'
#' @examples
add_solicitation_group <-
function(data) {
if (!data %>% hasName("codeProductService")) {
return(data)
}
data <-
data %>%
mutate(
idSolicitationGroup =
case_when(
is.na(codeProductService) ~ NA_character_,
codeProductService %>% substr(1, 1) %>% str_detect("[A-Z]") ~ codeProductService %>% substr(1, 1),
TRUE ~ codeProductService %>% substr(1, 2)
)
)
df_psc <-
dictionary_psc_active() %>% distinct(idSolicitationGroup, nameSolicitationGroup)
data %>%
select(-matches("nameSolicitationGroup")) %>%
left_join(df_psc, by = "idSolicitationGroup")
}
# bulk --------------------------------------------------------------------
#' Bulk FPDS Download
#'
#' Download data from FPDS for specified
#' years
#'
#' @param years vector of years from 1970 until present
#' @param path file path
#' @param split_amounts if \code{TRUE} split amounts for search
#'
#' @return \code{tibble()}
#' @export
#'
#' @examples
bulk_download_fpds <-
function(years = 1977:2000,
path = "Desktop/data/usa_spending/fpds",
split_amounts = T,
return_message = F,
snake_names = T) {
oldwd <- getwd()
setwd("~")
years %>%
walk(function(year) {
year %>% message()
data <-
bulk_fpds_csv_years(
years = year,
split_amounts = split_amounts,
return_message = return_message
)
data_path <- glue("{path}/{year}.tsv.gz")
data <- data %>%
munge_lite(unformat = T, snake_names = snake_names)
data %>% write_csv(data_path)
rm(data)
gc()
return(invisible())
})
}
# bulk_atom ---------------------------------------------------------------
.atom_year_url_old <-
function(year = 1979, splits = 4) {
df_urls <-
year %>%
map_dfr(function(year) {
if (year == 1977) {
start <- "1900-01-01"
end <- "1977-12-31"
}
if (year != 1977) {
start <- glue("{year}-01-01")
end <- glue("{year}-12-31")
}
fpds_atom(signed_date = c(start, end)) %>%
mutate(yearData = year) %>%
select(yearData, everything())
})
matches <- nrow(df_urls) * 10 %>% formattable::comma(digits = 0)
glue("\n\n{red({matches})} procurement actions for years: {blue(year)}\n\n") %>% message()
splits <- case_when(year == 1977 ~ 4,
TRUE ~ splits)
df_urls <- df_urls %>%
mutate(group = yearData %>% ntile(n = splits))
df_urls
}
.atom_year_url <-
function(year = 2019, splits = 10) {
df_urls <-
year %>%
map_dfr(function(year) {
if (year == 1977) {
start <- "1900-01-01"
end <- "1977-12-31"
data <-
fpds_atom(signed_date = c(start, end)) %>%
mutate(yearData = year) %>%
select(yearData, everything())
return(data)
}
df_months <-
.year_month_normal(year = year)
all_data <-
1:nrow(df_months) %>%
map_dfr(function(x){
glue("Period {x} for {year}") %>% message()
df_m <- df_months[x,]
start <-df_m$dateStart
end <- df_m$dateEnd
data <-
fpds_atom(signed_date = c(start, end)) %>%
mutate(yearData = year) %>%
select(yearData, everything())
data
})
all_data
})
matches <- nrow(df_urls) * 10 %>% formattable::comma(digits = 0)
glue("\n\n{red({matches})} procurement actions for years: {blue(year)}\n\n") %>% message()
splits <- case_when(year == 1977 ~ 4,
TRUE ~ splits)
df_urls <-
df_urls %>%
mutate(group = yearData %>% ntile(n = splits))
df_urls
}
.consolidate_year_rda <-
function(path = "Desktop/data/usa_spending/fpds_atom",
year = 1978,
exclude_bloat = T) {
oldwd <- getwd()
setwd("~")
final_path <-
glue("{path}/{year}")
final_file <-
glue("{year}_final.rda")
setwd(final_path)
files <-
list.files()[list.files() %>% str_detect(".rda")] %>%
discard(function(x) {
x %>% str_detect("final")
})
data <-
files %>% map_dfr(read_rda)
names(data) <-
names(data) %>% str_replace_all("_office_contracting", "office_award")
add_vendor <- names(data)[names(data) %in% c(
"city",
"code_congressional_district",
"code_country",
"code_state",
"zipcode",
"name_state",
"address_street2",
"name_country",
"fax",
"telephone",
"address_street1",
"country"
)]
if (length(add_vendor) > 0) {
names(data)[names(data) %in% add_vendor] <-
names(data)[names(data) %in% add_vendor] %>% str_c("_vendor")
}
if (exclude_bloat) {
data <-
data %>%
select(-matches("url_fpds|names_vendor"))
}
data <- data %>%
mutate_at(data %>% select(matches("id_duns")) %>% names(),
list(function(x) {
if_else(x == 0, NA_real_, x)
}))
data <-
data %>%
mutate(is_contractor_not_vendor = name_contractor != name_vendor)
v_n <-
data %>% select(matches("name_vendor|name_contractor")) %>% names()
data <-
data %>%
mutate_at(v_n, list(function(x) {
if_else(x == "NO DATA FROM D AND B", NA_character_, x)
}))
data <- data %>% mutate_if(is.numeric, as.numeric)
### deal with counts
data %>% save(file = final_file)
files %>%
walk(function(file) {
if (file.exists(file))
#Delete file if it exists
file.remove(file)
})
if (getwd() != oldwd) {
setwd("~")
setwd(oldwd)
}
glue("\n\nFINISHED {crayon::red(year)}\n\n\n") %>% message()
return(invisible())
}
#' Consolidate bulk bulk .rda files
#'
#' @param path file path
#' @param years vector of sub-folder years
#' @param exclude_bloat if \code{TRUE} excludes bloat columns
#'
#' @return
#' @export
#'
#' @examples
consolidate_atom_years_rda <-
function(path = "Desktop/data/usa_spending/fpds_atom",
years = 1978,
exclude_bloat = T) {
years %>%
future_walk(function(year) {
.consolidate_year_rda(path = path,
year = year,
exclude_bloat = exclude_bloat)
}, .progress = T)
}
#' FPDS Atom URL Dictionaries
#'
#' @param years vector of years
#' @param splits number of splits
#'
#' @return
#' @export
#'
#' @examples
fpds_years_atom_urls <-
function(years = 1979:2020,
splits = 4) {
data <-
years %>%
map_dfr(function(year) {
.atom_year_url(year = year, splits)
})
data
}
.fpds_atom_year_bulk <-
function(year = 1977,
strategy = "multisession",
splits = 5,
start_split = NULL,
end_split = NULL,
clean_entity_column = F,
unformat = T,
keep_key_columns = F,
show_progress = T,
path = "Desktop/data/usa_spending/fpds_atom",
snake_names = T,
clean_address = T,
return_message = T) {
df_urls <-
fpds_years_atom_urls(years = year, splits = splits)
if (length(start_split) > 0) {
df_urls <-
df_urls %>%
filter(group >= start_split)
}
oldwd <- getwd()
setwd("~")
has_year <-
path %>% list.dirs() %>% str_detect(as.character(year)) %>% sum(na.rm = T) == 1
final_path <- glue("{path}/{year}") %>% as.character()
if (!has_year) {
dir.create(path = final_path)
}
first_group <- df_urls$group %>% min()
if (length(end_split) > 0) {
splits <- end_split
}
first_group:splits %>%
walk(function(x) {
urls <-
df_urls %>%
filter(group == x) %>%
pull(urlFPDSAtom)
x <- as.character(x)
num_slug <-
case_when(
nchar(x) == 1 ~ glue("000{x}") %>% as.character(),
nchar(x) == 2 ~ glue("00{x}") %>% as.character(),
nchar(x) == 3 ~ glue("0{x}") %>% as.character(),
TRUE ~ as.character(x)
)
file_name <-
glue("{year}_{num_slug}.rda") %>% as.character()
data_path <-
glue("{final_path}/{file_name}") %>% as.character()
if (length(strategy) > 0) {
plan(strategy = strategy)
}
data <-
parse_fpds_atom_urls(
urls = urls,
clean_entity_column = clean_entity_column,
unformat = unformat,
keep_key_columns = keep_key_columns,
return_message = return_message,
exclude_bloat = T,
clean_address = clean_address,
snake_names = F,
use_future = T,
show_progress = show_progress
)
if (snake_names) {
data <- data %>%
janitor::clean_names()
}
data %>% save(file = data_path)
closeAllConnections()
gc()
return(invisible())
})
if (getwd() != oldwd) {
setwd(oldwd)
}
message("FINISHED")
gc()
return(invisible())
}
#' Bulk FPDS Download
#'
#' @param years vector of years
#' @param path file path
#' @param clean_entity_column if \code{TRUE} cleans entity columns
#' @param unformat if \code{TRUE} removes formattable formats
#' @param snake_names snake the names
#' @param return_message if \code{TRUE} returns message
#' @param splits if \code{TRUE} number of file splits
#' @param start_split
#' @param keep_key_columns
#' @param strategy future strategy \itemize{
#' \item sequential
#' \item multisession
#' \item cluster
#' }
#' @param clean_address
#'
#' @return
#' @export
#'
#' @examples
bulk_download_fpds_atom <-
function(years = 2001,
splits = 15,
end_split = NULL,
clean_entity_column = F,
unformat = T,
strategy = "multisession",
clean_address = T,
show_progress = T,
start_split = NULL,
keep_key_columns = F,
path = "Desktop/data/usa_spending/fpds_atom",
snake_names = T,
return_message = T) {
years %>%
walk(function(year_no) {
.fpds_atom_year_bulk(
year = year_no,
splits = splits,
clean_entity_column = clean_entity_column,
unformat = unformat,
strategy = strategy,
start_split = start_split,
keep_key_columns = keep_key_columns,
show_progress = show_progress,
path = path,
snake_names = snake_names,
clean_address = clean_address,
return_message = return_message,
end_split = end_split
)
})
closeAllConnections()
message("FINISHED")
gc()
return(invisible())
}
abresler/govtrackR documentation built on July 11, 2020, 12:30 a.m.
R Package Documentation
Browse R Packages
We want your feedback!
Note that we can't provide technical support on individual packages. You should contact the package authors for that.
Defines functions .read_fdps_csv .create_fpds_query_params .munge_double_list .create_date_slug .count_results .shorten_character .shorten_count .shorten_amounts .shorten_dates .create_value_slug .sort_slug .munge_fpds_names .munge_fdps_data pad_fdps_duns .pad_fdps_duns dictionary_fpds_search_parameters dictionary_fpds_parameter_description
Documented in dictionary_fpds_parameter_description dictionary_fpds_search_parameters pad_fdps_duns
# https://www.fpds.gov/wiki/index.php/FPDS-NG_FAQ
# https://www.fpds.gov/wiki/index.php/ATOM_Feed_FAQ
#' FPDS parameter description dictionary
#'
#' @return
#' @export
#'
#' @examples
dictionary_fpds_parameter_description <- function() {
page <-
"https://www.fpds.gov/wiki/index.php/EzSearch_FAQ" %>% read_html()
tables <- page %>% html_table(fill = F)
data <- tables[[1]] %>% as_tibble()
data <-
data %>% setNames(c(
"nameItem",
"parameterSearch",
"typeSearch",
"descriptionSearch"
))
data
}
# dictionaries ------------------------------------------------------------
#' FPDS Search Parameter Dictionary
#'
#' @return
#' @export
#'
#' @examples
#' dictionary_fpds_search_parameters()
dictionary_fpds_search_parameters <-
function() {
parameter <-
c(
"CONTRACT_STATUS_DESCRIPTION",
"VENDOR_SITE_CODE_ALTERNATE",
"PROGRAM_ACRONYM",
"AD_POST_AWARD_AUDIT",
"CURRENT_CONTRACT_VALUE",
"CONTRACT_TYPE_DESCRIPTION",
"AD_NONE",
"CONTRACTOR_NAME",
"CONTRACT_FINANCING_DESC",
"SECURITY_CODE",
"TRANSACTION_NUMBER",
"NGC_PARTICIPATION_DESC",
"REASON_NOT_COMPETED",
"RECOVERED_MATERIAL_CLS_DESC",
"ORIGINAL_PIID",
"PRINCIPAL_INV_FIRST_NAME",
"EMAIL_ADDRESS",
"PRODUCT_OR_SERVICE_CODE",
"LEGISLATIVE_MANDATES",
"SERVICE_CONTRACT_ACT",
"WALSH_HEALEY_ACT",
"DOMESTIC_VENDOR_NAME",
"DAVIS_BACON_ACT",
"VENDOR_RENEWAL_DATE",
"PARENT_MACOM_CODE",
"RESEARCH",
"UNDEFINITIZED_ACTION_DESC",
"DEPARTMENT_ID",
"TYPE_OF_SET_ASIDE_DESCRIPTION",
"AD_OTHER",
"FEE_PAID_FOR_USE_OF_SERVICE",
"PROPERTY_FINANCIAL_REPORTING",
"VENDOR_DUNS_NUMBER",
"WHO_CAN_USE_DESC",
"SOLICITATION_PROC_DESCRIPTION",
"AD_QUALITY_INSURANCE",
"FUNDING_OFFICE_NAME",
"CONTRACTING_AGENCY",
"SERVICE_CONTRACT_ACT_DESC",
"AWARD_STATUS",
"REASON_NOT_SB_DESCRIPTION",
"AD_CLOSEOUT",
"OBLIGATED_AMOUNT",
"REASON_NOT_COMP_DESCRIPTION",
"AGENCY_FULL_NAME",
"MULTIYEAR_CONTRACT",
"SYNOPSIS_WAIVER_EXCEPTION",
"DESCRIPTION_OF_REQUIREMENT",
"LAST_MODIFIED_BY",
"SUB_ACCOUNT_CODE",
"COMMERCIAL_ITEM_ACQUISTN_DESC",
"CONTRACT_BUNDLING",
"INHERENTLY_GOVERNMENTAL_DESC",
"NON_GOVERNMENT_VALUE",
"VENDOR_SITE_CODE",
"AD_ENGG_OR_PROD_SURVEILLANCE",
"PART8_OR_PART13",
"SBC_DEMONSTRATION_PROGRAM",
"ALTERNATIVE_ADVERTISING_DESC",
"CONSOLIDATED_CONTRACT_DESC",
"EVALUATED_PREF_DESCRIPTION",
"LOCATION_CODE",
"REF_IDV_CLASS_ID",
"PRINCIPAL_INV_LAST_NAME",
"ALT_PRINCIPAL_INV_MI_INITIAL",
"NAICS_DESCRIPTION",
"REF_IDV_AGENCY_ID",
"AD_TRANSPORTATION",
"MFG_ORGANIZATION_TYPE",
"SUBCONTRACT_PLAN",
"ORDERING_PROCEDURE",
"TYPE_OF_CONTRACT_PRICING",
"COST_ACCT_STANDARDS",
"ACCTNG_INSTALLATION_NUMBER",
"ADDITIONAL_REPORTING",
"MAIN_ACCOUNT_CODE",
"URL_OF_PROGRAM",
"ADMINISTRATOR_CODE",
"LOCAL_AREA_SET_ASIDE",
"AGENCY_NAME",
"CAS_CLAUSE_DESC",
"PRICE_EVAL_PERCENT_DIFF",
"ACCESSION_NUMBER",
"EXTENT_COMPETED_DESCRIPTION",
"FEE_RANGE_LOWER_VALUE",
"VENDOR_NAME",
"MULTIPLE_OR_SINGLE_DESC",
"TYPE_OF_FEE_FOR_USE_OF_SERVICE",
"AWARD_TYPE_DESCRIPTION",
"MAJOR_PROGRAM_CODE",
"PBSC_SHORT_DESCRIPTION",
"LASA_DESCRIPTION",
"CCR_EXCEPTION_TYPE_DESC",
"SOLICITATION_DATE",
"AD_BLANKET_DELEGATION",
"SYSTEM_EQUIPMENT_DESCRIPTION",
"INSTALLAION_UNIQUE",
"COMMERCIAL_ITEM_TEST_PROGRAM",
"COUNTRY_OF_INCORPORATION",
"CONTINGENCY_OPS_DESC",
"FINAL_INVOICE_PAID_DATE",
"MFG_ORGANIZATION_TYPE_DESC",
"TOTAL_DOLLARS_OBLIGATED",
"NASA_OTHER_STATUTORY_AUTHORITY",
"DOMESTIC_DUNS_NUMBER",
"REASON_FOR_MODIFICATION_DESC",
"FEE_RANGE_UPPER_VALUE",
"VENDOR_DOMESTIC_DUNS_NUMBER",
"FUNDED_BY_FOREIGN_ENTITY",
"CLINGER_COHEN_ACT_DESC",
"NUMBER_OF_OFFERS_RECEIVED",
"SEA_TRANSPORTATION_DESC",
"SOCIO_ECONOMIC_INDICATORS",
"ANNUAL_REVENUE",
"CONTRACT_TYPE",
"MGMT_REPORTING_REQUIREMENTS",
"AD_CONSENT_TO_SUBCONTRACT",
"NEW_TECH_PATENT_RIGHTS_CLAUSE",
"VENDOR_ADDRESS_STATE_NAME",
"PRODUCT_OR_SERVICE_DESCRIPTION",
"AWARD_COUNTRY_NAME",
"GLOBAL_VENDOR_NAME",
"LOCATION_NAME",
"LOCATION_STATE_NAME",
"SYS_LAST_MODIFIED_DATE",
"AWARD_TYPE",
"COST_OR_PRICING_DATA_DESC",
"COUNTY_NAME",
"CREATED_VIA",
"NUMBER_OF_ACTIONS",
"POM_SHORT_DESCRIPTION",
"POP_CONGRESS_DISTRICT_CODE",
"EXTENT_COMPETED",
"REF_IDV_PIID",
"REASON_NOT_SB",
"FED_BIZ_OPPS",
"PRODUCT_OR_SERVICE_TYPE",
"FUNDING_AGENCY_NAME",
"SOLICITATION_ID",
"ULTIMATE_CONTRACT_VALUE",
"PRIMARY_KEY",
"PRINCIPAL_NAICS_CODE",
"CONTRACT_FISCAL_YEAR",
"NATIONAL_INTEREST_CODE",
"TOTAL_ULTIMATE_CONTRACT_VALUE",
"TYPE_OF_FEE_DESC",
"ALT_PRINCIPAL_INV_FIRST_NAME",
"REGION_CODE",
"REASON_FOR_MODIFICATION",
"SIGNED_DATE",
"COUNTRY_OF_INCORP_NAME",
"DOLLARS_OBLIGATED",
"CLOSEOUT_PR",
"PIID",
"IT_COMMERCIAL_ITEM_CAT_DESC",
"SPENDING_CATEGORY",
"REF_IDV_AGENCY_NAME",
"PR_NUMBER",
"COTR_NAME",
"TYPE_OF_IDC_DESC",
"GLOBAL_DUNS_NUMBER",
"WHO_CAN_USE",
"FAIR_OPPORTUNITY_DESCRIPTION",
"FUNDED_THROUGH_DATE",
"FUNDING_DEPARTMENT_ID",
"SBC_DEMONSTRATION_PROGRAM_DESC",
"VALUE_ENGINEERING_CLAUSE",
"IT_COMMERCIAL_ITEM_CATEGORY",
"BASE_EXERCISED_OPTIONS_VALUE",
"TYPE_OF_SET_ASIDE",
"SOLICITATION_PROCEDURES",
"STATE_OF_INCORP_NAME",
"FUNDING_AGENCY_ID",
"PARENT_DUNS_NUMBER",
"VENDOR_ADDRESS_STATE_CODE",
"IS_PHYSICALLY_COMPLETE",
"CONTRACT_FUND_CODE",
"LOCATION_STATE_CODE",
"AGENCY_CODE",
"DEPARTMENT_FULL_NAME",
"AWARD_CLASS_ID",
"VENDOR_PHONE_NUMBER",
"ESTIMATED_COMPLETION_DATE",
"LAST_MODIFIED_DATE",
"TYPE_OF_IDC",
"VENDOR_PARENT_DUNS_NUMBER",
"TYPE_OF_AGREEMENT",
"ALTERNATE_COTR_NAME",
"IS_TRANSFERED",
"AD_COST_ACCOUNTING_STANDARDS",
"PROCUREMENT_PLACEMENT_CODE",
"FISCAL_YEAR",
"COUNTRY_CODE",
"CONTRACT_BUNDLING_DESC",
"EFFECTIVE_DATE",
"CANCELLATION_DATE",
"DAVIS_BACON_ACT_DESC",
"FUNDING_OFFICE_ID",
"LAST_DATE_TO_ORDER",
"VENDOR_ADDRESS_COUNTRY_CODE",
"PLACE_OF_PERFORMANCE_ZIP_CODE",
"PLACE_OF_PERFORM_COUNTRY_CODE",
"EVALUATED_PREFERENCE",
"PHYSICAL_COMPLETION_DATE",
"ADVISORY_ASSISTANCE_SERVICES",
"OTHER_STATUTORY_AUTHORITY",
"VENDOR_ADDRESS_ZIP_CODE",
"VENDOR_ADDRESS_COUNTRY_NAME",
"VENDOR_ULTIMATE_DUNS_NUMBER",
"VENDOR_ADDRESS_LINE_2",
"VENDOR_ADDRESS_LINE_1",
"VENDOR_ADDRESS_LINE_3",
"COUNTY_CODE",
"POP_STATE_NAME",
"ORGANIZATION_CODE",
"POP_COUNTRY_NAME",
"CONTINGENCY_OPS",
"SOLICITATION_ISSUE_DATE",
"CLAIMANT_PROGRAM",
"CONTRACTING_AUTHORITY_DESC",
"TOTAL_NON_GOVERNMENT_VALUE",
"MULTIPLE_OR_SINGLE_AWARD_IDC",
"VERSION",
"VENDOR_CONGRESS_DISTRICT_CODE",
"CLAIMANT_PROGRAM_DESCRIPTION",
"FED_BIZ_OPPS_DESC",
"CONTRACT_SUB_TYPE",
"A76_ACTION",
"STATE_OF_INCORPORATION",
"CONTRACTING_AGENCY_NAME",
"NEGOTIATOR_CODE",
"CONTRACT_FINANCING",
"RECOVERED_MATERIAL_CLAUSES",
"VENDOR_DIVISION_NUMBER",
"CREATED_DATE",
"PCARD_AS_PAYMENT_DESC",
"REASON_NOT_SDB_DESCRIPTION",
"VENDOR_ADDRESS_CITY",
"AGENCY_IDENTIFIER",
"TREASURY_ACCOUNT_SYMBOL",
"ALT_PRINCIPAL_INV_LAST_NAME",
"CONTRACTING_AUTHORITY",
"PRINCIPAL_INV_MIDDLE_INITIAL",
"DESTROY_DATE",
"CONSOLIDATED_CONTRACT",
"MULTIYEAR_CONTRACT_DESC",
"PURCHASE_REASON",
"VENDOR_DOING_BUSINESS_AS_NAME",
"SUBCONTRACT_PLAN_DESC",
"CONTRACTING_OFFICER_CODE",
"AWARD_COMPLETION_DATE",
"LETTER_CONTRACT",
"FUNDING_DEPARTMENT_NAME",
"FIXED_FEE_VALUE",
"TOTAL_CURRENT_CONTRACT_VALUE",
"PRE_AWARD_SYNOPSIS_DESC",
"GFE_GFP",
"ALTERNATIVE_ADVERTISING",
"CREATED_BY",
"CLINGER_COHEN_ACT",
"SYSTEM_EQUIPMENT_CODE",
"LAST_MOD_DATE",
"VENDOR_REGISTRATION_DATE",
"CONTRACT_PRICING_DESCRIPTION",
"ORIGINAL_AGENCY_CODE",
"FOREIGN_FUNDING_DESC",
"A76_DESC",
"VENDOR_FAX_NUMBER",
"COMMERCIAL_ITEM_ACQUISITION",
"VENDOR_FULL_NAME",
"PARENT_VENDOR_NAME",
"SUPPORT_SERVICES_TYPE_CONTRACT",
"ADDITIONAL_REPORTING_DESC",
"REASON_FOR_PURCHASE_DESC",
"RESEARCH_DESCRIPTION",
"OFFERORS_PROPOSAL_NUMBER",
"PCARD_AS_PAYMENT",
"VENDOR_DIVISION_NAME",
"DEPARTMENT_NAME",
"REF_IDV_MODIFICATION_NUMBER",
"CITP_DESC",
"FIELD_OF_SCIENCE_OR_ENGG",
"PLACE_OF_MANUFACTURE",
"ORGANIZATIONAL_TYPE",
"CCR_EXCEPTION_TYPE",
"WALSH_HEALEY_ACT_DESC",
"REVEAL_DATE",
"DOCUMENT_TYPE",
"GFE_GFP_DESC",
"CONTRACTING_OFFICE_NAME",
"USE_OF_EPA_PRODUCTS_DESC",
"USE_OF_EPA_PRODUCTS",
"PRE_AWARD_SYNOPSIS_REQUIREMENT",
"INHERENTLY_GOVERNMENTAL_CODE",
"MAXIMUM_ORDER_LIMIT",
"NATIONAL_INTEREST_DESCRIPTION",
"CO_BUS_SIZE_DETERMINATION_DESC",
"CO_BUS_SIZE_DETERMINATION",
"AD_SECURITY",
"CONTRACTING_AGENCY_ID",
"VENDOR_NAME_ALTERNATE",
"SEA_TRANSPORTATION",
"SWE_SHORT_DESCRIPTION",
"CONTRACTING_OFFICE_ID",
"STATRY_EXPTN_TO_FAIR_OPRTNTY",
"PERFORM_BASED_SERVICE_CONTRACT",
"AD_PROPERTY_ADMINISTRATION",
"COST_OR_PRICING_DATA",
"REASON_NOT_SDB",
"VENDOR_LEGAL_ORGANIZATION_NAME",
"NUMBER_OF_EMPLOYEES",
"MODIFICATION_NUMBER",
"NGC_PARTICIPATION"
)
parameter_name <-
c(
"contract_status_description",
"vendor_site_code_alternate",
"program_acronym",
"ad_post_award_audit",
"current_contract_value",
"award_type",
"ad_none",
"contractor_name",
"contract_financing_desc",
"security_code",
"transaction_number",
"nontraditional_government_contractor_participation",
"reason_not_competed",
"recovered_material_cls_desc",
"original_piid",
"principal_inv_first_name",
"email_address",
"psc",
"legislative_mandates",
"service_contract_act",
"walsh_healey_act",
"domestic_vendor_name",
"davis_bacon_act",
"vendor_renewal_date",
"parent_macom_code",
"research",
"undefinitized_action_desc",
"department_id",
"type_of_set_aside",
"ad_other",
"fee_paid_for_use_of_service",
"property_financial_reporting",
"vendor_duns",
"who_can_use_desc",
"solicitation_procedures",
"ad_quality_insurance",
"funding_office_name",
"contracting_agency",
"service_contract_act_desc",
"award_status",
"reason_not_sb_description",
"ad_closeout",
"action_obligation",
"other_than_full_and_open_competition",
"agency_full_name",
"multiyear_contract",
"synopsis_waiver_exception",
"description_of_requirement",
"last_modified_by",
"treasury_account_symbol_sub_account_code",
"commercial_item_acquisition_procedures",
"contract_bundling",
"inherently_governmental_description",
"non-government_dollars",
"vendor_site_code",
"ad_engg_or_prod_surveillance",
"part8_or_part13",
"small_business_competitiveness_demonstration_program",
"alternative_advertising_desc",
"consolidated_contract_desc",
"evaluated_preference",
"location_code",
"ref_idv_class_id",
"principal_inv_last_name",
"alt_principal_inv_mi_initial",
"naics_description",
"reference_idv_agency_id",
"ad_transportation",
"mfg_organization_type",
"subcontract_plan",
"ordering_procedure",
"type_of_contract_pricing",
"cost_acct_standards",
"acctng_installation_number",
"additional_reporting_code",
"treasury_account_symbol_main_account_code",
"url_of_program",
"administrator_code",
"local_area_set_aside",
"agency_name",
"cas_clause_desc",
"price_eval_percent_diff",
"accession_number",
"extent_competed",
"fee_range_lower_value",
"vendor_name",
"multiple_or_single_desc",
"type_of_fee_for_use_of_service",
"award_type",
"major_program_code",
"pbsc_short_description",
"lasa_description",
"ccr_exception_type_desc",
"solicitation_date",
"ad_blanket_delegation",
"system_equipment_description",
"installaion_unique",
"commercial_item_test_program",
"country_of_incorporation",
"contingency_ops_desc",
"final_invoice_paid_date",
"mfg_organization_type_desc",
"total_dollars_obligated",
"nasa_other_statutory_authority",
"domestic_duns_number",
"reason_for_modification_desc",
"fee_range_upper_value",
"vendor_domestic_duns_number",
"funded_by_foreign_entity",
"clinger_cohen_act_desc",
"number_of_offers_received",
"sea_transportation_desc",
"socio_economic_indicators",
"annual_revenue",
"contract_type",
"mgmt_reporting_requirements",
"ad_consent_to_subcontract",
"new_tech_patent_rights_clause",
"vendor_state_name",
"psc_description",
"award_country_name",
"global_vendor_name",
"location_name",
"location_state_name",
"sys_last_modified_date",
"award_type",
"cost_or_pricing_data_desc",
"county_name",
"created_via",
"number_of_actions",
"pom_short_description",
"pop_congressional_district_code",
"extent_competed",
"reference_idv",
"reason_not_sb",
"fed_biz_opps",
"psc_type",
"funding_agency_name",
"solicitation_id",
"ultimate_contract_value",
"primary_key",
"naics",
"contract_fiscal_year",
"national_interest_code",
"total_base_and_exercised_options_value",
"type_of_fee_desc",
"alt_principal_inv_first_name",
"region_code",
"reason_for_modification",
"date_signed",
"country_of_incorp_name",
"dollars_obligated",
"closeout_pr",
"contract_id",
"it_commercial_item_cat_desc",
"initiative",
"ref_idv_agency_name",
"pr_number",
"cotr_name",
"type_of_idc_desc",
"global_duns_number",
"who_can_use",
"fair_opportunity",
"funded_through_date",
"funding_department_id",
"small_business_competitiveness_demonstration_program",
"value_engineering_clause",
"it_commercial_item_category",
"base_and_exercised_options_value",
"type_of_set_aside",
"solicitation_procedures",
"state_of_incorp_name",
"funding_agency_id",
"parent_duns_number",
"vendor_state",
"is_physically_complete",
"contract_fund_code",
"location_state_code",
"agency_code",
"department_full_name",
"award_class_id",
"vendor_phone_number",
"estimated_completion_date",
"last_modified_date",
"type_of_idc",
"vendor_parent_duns_number",
"type_of_agreement",
"alternate_cotr_name",
"is_transfered",
"ad_cost_accounting_standards",
"procurement_placement_code",
"other_transaction_fiscal_year",
"country_code",
"contract_bundling_desc",
"effective_date",
"cancellation_date",
"davis_bacon_act_desc",
"funding_office_id",
"last_date_to_order",
"vendor_country_code",
"place_of_performance_zip_code",
"place_of_perform_country_code",
"evaluated_preference",
"physical_completion_date",
"advisory_assistance_services",
"other_statutory_authority",
"vendor_zip_code",
"vendor_country_name",
"vendor_ultimate_duns_number",
"vendor_address_line_2",
"vendor_address_line_1",
"vendor_address_line_3",
"county_code",
"pop_state_name",
"organization_code",
"pop_country_name",
"contingency_ops",
"solicitation_issue_date",
"claimant_program",
"contracting_authority_desc",
"total_non_government_value",
"multiple_or_single_award_idc",
"version",
"vendor_congressional_district_code",
"claimant_program_description",
"fed_biz_opps",
"award_type",
"a76_action",
"state_of_incorporation",
"contracting_agency_name",
"negotiator_code",
"contract_financing",
"recovered_material_clauses",
"vendor_division_number",
"created_date",
"pcard_as_payment_desc",
"reason_not_sdb_description",
"vendor_city",
"treasury_account_symbol_agency_identifier",
"treasury_account_symbol",
"alt_principal_inv_last_name",
"contracting_authority",
"principal_inv_middle_initial",
"destroy_date",
"consolidated_contract",
"multiyear_contract_desc",
"purchase_reason",
"vendor_doing_business_as_name",
"subcontract_plan_desc",
"contracting_officer_code",
"award_completion_date",
"letter_contract",
"funding_department_name",
"fixed_fee_value",
"total_base_and_all_options_value",
"pre_award_synopsis_desc",
"gfe_gfp",
"alternative_advertising",
"created_by",
"clinger_cohen_act",
"system_equipment_code",
"last_modified_date",
"vendor_registration_date",
"contract_pricing_description",
"original_agency_code",
"foreign_funding_desc",
"a76",
"vendor_fax_number",
"commercial_item_acquisition",
"vendor_full_name",
"parent_vendor_name",
"support_services_type_contract",
"additional_reporting_description",
"reason_for_purchase_desc",
"sbir_sttr",
"offerors_proposal_number",
"pcard_as_payment",
"vendor_division_name",
"department_name",
"reference_idv_modification_number",
"commercial_item_test_program",
"field_of_science_or_engg",
"place_of_manufacture",
"organizational_type",
"ccr_exception_type",
"walsh_healey_act_desc",
"reveal_date",
"document_type",
"gfe_gfp_desc",
"contracting_office_name",
"use_of_epa_products_desc",
"use_of_epa_products",
"pre_award_synopsis_requirement",
"inherently_governmental_code",
"maximum_order_limit",
"national_interest_description",
"co_bus_size_determination_desc",
"co_bus_size_determination",
"ad_security",
"contracting_agency_id",
"vendor_name_alternate",
"sea_transportation",
"swe_short_description",
"contracting_office_id",
"statry_exptn_to_fair_oprtnty",
"perform_based_service_contract",
"ad_property_administration",
"cost_or_pricing_data",
"reason_not_sdb",
"vendor_legal_organization_name",
"number_of_employees",
"modification_number",
"ngc_participation"
)
tibble(parameter, parameter_name)
}
# https://www.fpds.gov/downloads/Version_1.4_specs/FPDSNG_DataDictionary_V1.4.pdf
#' FPDS name dictionary
#'
#' @return
#' @export
#'
#' @examples
dictionary_fpds_names <-
memoise::memoise(function() {
nameFPDS = c(
"ref_idv_contract_id",
"parent_vendor_name",
"subcontract_plan",
"solicitation_proc_description",
"award_type",
"awardIDagencyID",
"awardIDagencyIDDescription",
"awardIDPIID",
"awardIDmodNumber",
"awardIDtransactionNumber",
"relevantContractDatessignedDate",
"relevantContractDateseffectiveDate",
"relevantContractDatescurrentCompletionDate",
"relevantContractDatesultimateCompletionDate",
"dollarValuesobligatedAmount",
"dollarValuesbaseAndExercisedOptionsValue",
"dollarValuesbaseAndAllOptionsValue",
"totalDollarValuestotalObligatedAmount",
"totalDollarValuestotalBaseAndExercisedOptionsValue",
"totalDollarValuestotalBaseAndAllOptionsValue",
"purchaserInformationcontractingOfficeAgencyID",
"purchaserInformationcontractingOfficeAgencyIDDescription",
"purchaserInformationcontractingOfficeID",
"purchaserInformationcontractingOfficeIDDescription",
"purchaserInformationfundingRequestingAgencyID",
"purchaserInformationfundingRequestingAgencyIDDescription",
"purchaserInformationfundingRequestingOfficeID",
"purchaserInformationfundingRequestingOfficeIDDescription",
"purchaserInformationforeignFunding",
"purchaserInformationforeignFundingDescription",
"contractMarketingDatafeePaidForUseOfService",
"contractDatacontractActionType",
"contractDatacontractActionTypeDescription",
"contractDatatypeOfContractPricing",
"contractDatatypeOfContractPricingDescription",
"contractDatanationalInterestActionCode",
"contractDatanationalInterestActionCodeDescription",
"contractDatadescriptionOfContractRequirement",
"contractDatainherentlyGovernmentalFunction",
"contractDatainherentlyGovernmentalFunctionDescription",
"contractDataGFEGFP",
"contractDataGFEGFPDescription",
"contractDataseaTransportation",
"contractDataseaTransportationDescription",
"contractDataundefinitizedAction",
"contractDataundefinitizedActionDescription",
"contractDataconsolidatedContract",
"contractDataconsolidatedContractDescription",
"contractDataperformanceBasedServiceContract",
"contractDataperformanceBasedServiceContractDescription",
"contractDatacontingencyHumanitarianPeacekeepingOperation",
"contractDatacontingencyHumanitarianPeacekeepingOperationDescription",
"contractDatalistOfTreasuryAccounts",
"contractDatacontractFinancing",
"contractDatacontractFinancingDescription",
"contractDatapurchaseCardAsPaymentMethod",
"contractDatapurchaseCardAsPaymentMethodDescription",
"contractDatanumberOfActions",
"legislativeMandatesClingerCohenAct",
"legislativeMandatesClingerCohenActDescription",
"legislativeMandatesmaterialsSuppliesArticlesEquipment",
"legislativeMandatesmaterialsSuppliesArticlesEquipmentDescription",
"legislativeMandateslaborStandards",
"legislativeMandateslaborStandardsDescription",
"legislativeMandatesconstructionWageRateRequirements",
"legislativeMandatesconstructionWageRateRequirementsDescription",
"legislativeMandatesinteragencyContractingAuthority",
"productOrServiceInformationproductOrServiceCode",
"legislativeMandatesinteragencyContractingAuthorityDescription",
"productOrServiceInformationcontractBundling",
"productOrServiceInformationproductOrServiceCodeDescription",
"productOrServiceInformationclaimantProgramCode",
"productOrServiceInformationcontractBundlingDescription",
"productOrServiceInformationprincipalNAICSCode",
"productOrServiceInformationclaimantProgramCodeDescription",
"productOrServiceInformationrecoveredMaterialClauses",
"productOrServiceInformationprincipalNAICSCodeDescription",
"productOrServiceInformationmanufacturingOrganizationType",
"productOrServiceInformationrecoveredMaterialClausesDescription",
"productOrServiceInformationsystemEquipmentCode",
"productOrServiceInformationmanufacturingOrganizationTypeDescription",
"productOrServiceInformationinformationTechnologyCommercialItemCategory",
"productOrServiceInformationsystemEquipmentCodeDescription",
"productOrServiceInformationuseOfEPADesignatedProducts",
"productOrServiceInformationinformationTechnologyCommercialItemCategoryDescription",
"productOrServiceInformationcountryOfOrigin",
"productOrServiceInformationuseOfEPADesignatedProductsDescription",
"productOrServiceInformationplaceOfManufacture",
"productOrServiceInformationcountryOfOriginDescription",
"vendorisAlaskanNativeOwnedBusinessCorporationOrFirm",
"productOrServiceInformationplaceOfManufactureDescription",
"vendorisAmericanIndianOwnedBusinessBusinessBusiness",
"vendorisIndianTribe",
"vendorisNativeHawaiianOwnedBusinessOrganizationOrFirm",
"vendorisTriballyOwnedBusinessBusinessFirm",
"vendorisSmallBusiness",
"vendorisVeteranOwnedBusiness",
"vendorisServiceDisabledVeteranOwnedBusinessBusiness",
"vendorisWomanOwned",
"vendorminorityOwned",
"vendorisVerySmallBusiness",
"vendorisWomanOwnedSmallBusiness",
"vendorisEconomicallyDisadvantagedWomanOwnedSmallBusiness",
"vendorisJointVentureWomenOwnedSmallBusiness",
"vendorisJointVentureWomenOwnedSmallBusiness",
"vendorisCommunityDevelopedCorporationOwnedFirm",
"vendorisLaborSurplusAreaFirm",
"vendorfederalGovernment",
"vendorisStateGovernment",
"vendorlocalGovernment",
"vendorisTribalGovernment",
"vendorisForeignGovernment",
"vendorbusinessOrOrganizationType",
"vendorisArchitectureAndEngineering",
"vendorisCommunityDevelopmentCorporation",
"vendorisConstructionFirm",
"vendorisDomesticShelter",
"vendorisEducationalInstitution",
"vendorisFoundation",
"vendorisHospital",
"vendorisManufacturerOfGoods",
"vendorisResearchAndDevelopment",
"vendorisServiceProvider",
"vendorisVeterinaryHospital",
"vendorisHispanicServicingInstitution",
"vendorreceivesContracts",
"vendorreceivesGrants",
"vendorreceivesContractsAndGrants",
"vendorisAirportAuthority",
"vendorisCouncilOfGovernments",
"vendorisHousingAuthoritiesPublicOrTribal",
"vendorisInterstateEntity",
"vendorisPlanningCommission",
"vendorisPortAuthority",
"vendorisTransitAuthority",
"vendorisSubchapterSCorporation",
"vendorisLimitedLiabilityCorporation",
"vendorisForeignOwnedAndLocated",
"vendorprofitStructure",
"vendorisShelteredWorkshop",
"vendorstateOfIncorporation",
"vendorcountryOfIncorporation",
"vendorstateOfIncorporationDescription",
"vendororganizationalType",
"vendorcountryOfIncorporationDescription",
"vendoris1862LandGrantCollege",
"vendoris1890LandGrantCollege",
"vendoris1994LandGrantCollege",
"vendorisHistoricallyBlackCollegeOrUniversity",
"vendorisMinorityInstitution",
"vendorisPrivateUniversityOrCollege",
"vendorisSchoolOfForestry",
"vendorisStateControlledInstitutionOfHigherLearning",
"vendorisTribalCollege",
"vendorisVeterinaryCollege",
"vendorisAlaskanNativeServicingInstitution",
"vendorisNativeHawaiianServicingInstitution",
"vendorisDOTCertifiedDisadvantagedBusinessEnterprise",
"vendorisSelfCertifiedSmallDisadvantagedBusiness",
"vendorisSBACertifiedSmallDisadvantagedBusiness",
"vendorisSBACertified8AProgramParticipant",
"vendorisSelfCertifiedHUBZoneJointVenture",
"vendorisSBACertifiedHUBZone",
"vendorisSBACertified8AJointVenture",
"vendorstreetAddress",
"vendorcity",
"vendorstate",
"vendorZIPCode",
"vendorstateDescription",
"vendorcountryCode",
"vendorZIPCodeDescription",
"vendorphoneNo",
"vendorcountryCodeDescription",
"vendorfaxNo",
"vendorcongressionalDistrictCode",
"vendortext",
"vendorDUNSNumber",
"vendorvendorName",
"vendorglobalParentDUNSNumber",
"vendorglobalParentDUNSName",
"vendorregistrationDate",
"vendorrenewalDate",
"placeOfPerformancestateCode",
"placeOfPerformancecountryCode",
"placeOfPerformanceplaceOfPerformanceZIPCode",
"placeOfPerformanceplaceOfPerformanceCongressionalDistrict",
"competitionextentCompeted",
"competitionsolicitationProcedures",
"competitionextentCompetedDescription",
"competitiontypeOfSetAside",
"competitionsolicitationProceduresDescription",
"competitionevaluatedPreference",
"competitiontypeOfSetAsideDescription",
"competitionresearch",
"competitionevaluatedPreferenceDescription",
"competitionnumberOfOffersReceived",
"competitionresearchDescription",
"competitioncommercialItemAcquisitionProcedures",
"competitioncommercialItemTestProgram",
"competitioncommercialItemAcquisitionProceduresDescription",
"competitionA76Action",
"competitioncommercialItemTestProgramDescription",
"competitionfedBizOpps",
"competitionA76ActionDescription",
"competitionlocalAreaSetAside",
"competitionfedBizOppsDescription",
"preferenceProgramssubcontractPlan",
"competitionlocalAreaSetAsideDescription",
"transactionInformationcreatedBy",
"preferenceProgramssubcontractPlanDescription",
"transactionInformationcreatedDate",
"transactionInformationlastModifiedBy",
"transactionInformationlastModifiedDate",
"transactionInformationstatus",
"transactionInformationapprovedBy",
"transactionInformationstatusDescription",
"transactionInformationapprovedDate",
"vendorisMinorityOwnedBusiness",
"vendorisSubcontinentAsianIndianAmericanOwnedBusiness",
"vendorisAsianPacificAmericanOwnedBusiness",
"vendorisBlackAmericanOwnedBusiness",
"vendorisHispanicAmericanOwnedBusiness",
"vendorisNativeAmericanOwnedBusiness",
"vendorisOtherMinorityOwned",
"vendorisFederalGovernment",
"vendorisFederallyFundedResearchAndDevelopmentCorp",
"vendorisFederalGovernmentAgency",
"vendorisLocalGovernment",
"vendorisCityLocalGovernment",
"vendorisCountyLocalGovernment",
"vendorisInterMunicipalLocalGovernment",
"vendorisLocalGovernmentOwned",
"vendorisMunicipalityLocalGovernment",
"vendorisSchoolDistrictLocalGovernment",
"vendorisTownshipLocalGovernment",
"vendorisCorporateEntityNotTaxExempt",
"vendorisCorporateEntityTaxExempt",
"vendorisPartnershipOrLimitedLiabilityPartnership",
"vendorisSolePropreitorship",
"vendorisSmallAgriculturalCooperative",
"vendorisInternationalOrganization",
"vendorisOtherbusinessOrOrganization",
"vendorisUSGovernmentEntity",
"vendorisForProfitOrganization",
"vendorisNonprofitOrganization",
"vendorisOtherNotForProfitOrganization",
"nameContract",
"urlFPDSContractAtom",
"datetimeContractModified",
"contractDatareasonForModification",
"contractDatareasonForModificationDescription",
"contractDatacostOrPricingData",
"contractDatacostOrPricingDataDescription",
"contractDatacostAccountingStandardsClause",
"contractDatacostAccountingStandardsClauseDescription",
"contractDatamultiYearContract",
"contractDatamultiYearContractDescription",
"placeOfPerformancelocationCode",
"competitionsmallBusinessCompetitivenessDemonstrationProgram",
"competitionpriceEvaluationPercentDifference",
"contractDatasolicitationID",
"legislativeMandateslistOfAdditionalReportingValues",
"vendorcageCode",
"competitiontypeOfSetAsideSource",
"competitiontypeOfSetAsideSourceDescription",
"competitionnumberOfOffersSource",
"competitionnumberOfOffersSourceDescription",
"transactionInformationclosedStatus",
"contractIDagencyID",
"contractIDagencyIDDescription",
"contractIDPIID",
"contractIDmodNumber",
"relevantContractDateslastDateToOrder",
"dollarValuestotalEstimatedOrderValue",
"contractMarketingDatawebsiteURL",
"contractMarketingDatawhoCanUse",
"contractMarketingDataemailAddress",
"contractMarketingDataindividualOrderLimit",
"contractDatamultipleOrSingleAwardIDC",
"contractDatamultipleOrSingleAwardIDCDescription",
"legislativeMandatesotherStatutoryAuthority",
"contractDatareferencedIDVMultipleOrSingle",
"contractDatareferencedIDVMultipleOrSingleDescription",
"contractDatareferencedIDVType",
"contractDatareferencedIDVTypeDescription",
"contractMarketingDatatypeOfFeeForUseOfService",
"contractMarketingDatatypeOfFeeForUseOfServiceDescription",
"contractDatamajorProgramCode",
"contractDatatypeOfIDC",
"contractDatatypeOfIDCDescription",
"contractDataprogramAcronym",
"contractMarketingDatafeeRangeLowerValue",
"contractMarketingDatafeeRangeUpperValue",
"contractMarketingDataorderingProcedure",
"competitionstatutoryExceptionToFairOpportunity",
"competitionstatutoryExceptionToFairOpportunityDescription",
"transactionInformationclosedBy",
"transactionInformationclosedDate",
"competitionreasonNotCompeted",
"competitionreasonNotCompetedDescription",
"contractDatareferencedIDVPart8OrPart13",
"vendorstreetAddress2",
"competitionidvTypeOfSetAside",
"competitionidvTypeOfSetAsideDescription",
"competitionidvNumberOfOffersReceived",
"genericTagsgenericStrings",
"agencySpecificAwardElementstext",
"agencySpecificAwardElementsPRNumber",
"agencySpecificAwardElementscloseoutPR",
"agencySpecificAwardElementsprocurementPlacementCode",
"agencySpecificAwardElementsadvisoryOrAssistanceServicesContract",
"agencySpecificAwardElementssupportServicesTypeContract",
"agencySpecificAwardElementsnewTechnologyOrPatentRightsClause",
"agencySpecificAwardElementsmanagementReportingRequirements",
"agencySpecificAwardElementspropertyFinancialReporting",
"agencySpecificAwardElementsvalueEngineeringClause",
"agencySpecificAwardElementssecurityCode",
"agencySpecificAwardElementsadministratorCode",
"agencySpecificAwardElementscontractingOfficerCode",
"agencySpecificAwardElementsnegotiatorCode",
"agencySpecificAwardElementsCOTRName",
"agencySpecificAwardElementsorganizationCode",
"agencySpecificAwardElementscontractFundCode",
"agencySpecificAwardElementsisPhysicallyComplete",
"agencySpecificAwardElementsaccountingInstallationNumber",
"agencySpecificAwardElementsfundedThroughDate",
"agencySpecificAwardElementsprincipalInvestigator",
"agencySpecificAwardElementsfieldOfScienceOrEngineering",
"agencySpecificAwardElementsphysicalCompletionDate",
"agencySpecificAwardElementsaccessionNumber",
"agencySpecificAwardElementsdestroyDate",
"agencySpecificAwardElementssolicitationIssueDate",
"agencySpecificAwardElementslistOfContractAdministrationsDelegated",
"agencySpecificAwardElementsfinalInvoicePaidDate",
"contractMarketingDatafixedFeeValue",
"legislativeMandatesWalshHealyAct",
"legislativeMandatesWalshHealyActDescription",
"legislativeMandatesserviceContractAct",
"legislativeMandatesserviceContractActDescription",
"legislativeMandatesDavisBaconAct",
"legislativeMandatesDavisBaconActDescription",
"piid",
"ref_idv_piid",
"ref_idv_agency_id",
"modification_number",
"contracting_agency_id",
"contracting_agency_name",
"contracting_office_id",
"contracting_office_name",
"funding_agency_id",
"funding_office_id",
"funding_office_name",
"agency_code",
"agency_name",
"department_id",
"department_name",
"last_mod_date",
"last_modified_by",
"award_completion_date",
"created_date",
"signed_date",
"effective_date",
"estimated_completion_date",
"obligated_amount",
"ultimate_contract_value",
"type_of_contract_pricing",
"award_status",
"contract_type",
"created_by",
"description_of_requirement",
"reason_for_modification",
"legislative_mandates",
"local_area_set_aside",
"socio_economic_indicators",
"multiyear_contract",
"national_interest_code",
"national_interest_description",
"principal_naics_code",
"naics_description",
"product_or_service_code",
"product_or_service_description",
"pop_congress_district_code",
"pop_congress_country",
"pop_state_name",
"vendor_address_city",
"vendor_congress_district_code",
"vendor_address_country_code",
"vendor_address_country_name",
"vendor_duns_number",
"vendor_doing_business_as_name",
"vendor_name",
"vendor_address_state_code",
"vendor_address_state_name",
"vendor_address_zip_code",
"global_vendor_name",
"vendortreasuryAccountSymbol",
"agencySpecificAwardElementsinstallationUnique",
"Contract ID",
"Reference IDV",
"Modification Number",
"Transaction Number",
"Award/IDV Type",
"Action Obligation ($)",
"Date Signed",
"Solicitation Date",
"Contracting Agency ID",
"Contracting Agency",
"Contracting Office Name",
"PSC Type",
"PSC",
"PSC Description",
"NAICS",
"NAICS Description",
"Vendor DUNS",
"Vendor Name",
"Vendor City",
"Vendor State",
"Vendor ZIP Code",
"Global DUNS Number",
"Global Vendor Name",
"Additional Reporting Code",
"Additional Reporting Description",
"CGAC AGENCY CODE",
"FPDS DEPARTMENT ID",
"AGENCY NAME",
"AGENCY ABBREVIATION",
"REGISTERED IN BROKER",
"REGISTERED IN ASP",
"ORIGINAL SOURCE FOR CGAC or SubTier",
"SUBTIER CODE",
"SUBTIER NAME",
"SUBTIER ABBREVIATION",
"Admin Org Name",
"ADMIN_ORG",
"FREC",
"FREC Entity Description",
"SubTierAcronymSource",
"SubTier appears in data in FPDS or ASP?",
"REGISTERED IN ASP_1",
"AGENCY NAME (ORIGINAL)",
"SUBTIER NAME (ORIGINAL)",
"IS_FREC",
"MISSION",
"WEBSITE",
"ICON FILENAME",
"Agency Code",
"Agency Name",
"Bureau Code",
"Bureau Name",
"Account Code",
"Account Name",
"Treasury Agency Code",
"Subfunction Code",
"Subfunction Title",
"BEA Category",
"On- or Off- Budget",
"1976",
"TQ",
"1977",
"1978",
"1979",
"1980",
"1981",
"1982",
"1983",
"1984",
"1985",
"1986",
"1987",
"1988",
"1989",
"1990",
"1991",
"1992",
"1993",
"1994",
"1995",
"1996",
"1997",
"1998",
"1999",
"2000",
"2001",
"2002",
"2003",
"2004",
"2005",
"2006",
"2007",
"2008",
"2009",
"2010",
"2011",
"2012",
"2013",
"2014",
"2015",
"2016",
"2017",
"2018",
"2019",
"2020",
"2021",
"2022",
"pop_country_name",
"parent_duns_number",
"research",
"Funding Agency Name",
"Funding Agency ID",
"Place of Manufacture Description",
"Country of Product or Service Origin Code",
"PIID",
"Completion Date",
"NAICS Code",
"Product or Service Code",
"Product or Service Description",
"Vendor Country",
"DUNS Number",
"Type of Contract",
"Award or IDV Type",
"Base and All Options Value (Total Contract Value)",
"Action Obligation",
"Title",
"Sol #",
"Agency",
"Office",
"Location",
"Type",
"Orig. Type",
"Last Posting Date",
"Orig. Posting Date",
"Response Deadline",
"Archiving Policy",
"Archive Date",
"Set-aside Type",
"Classification Code",
"Naics Code",
"Office Address",
"Contact Info",
"Place of Performance",
"Additional Info Link",
"Description",
"Active",
"Link",
"Award #",
"Posting Date",
"Award Date",
"Award Amount",
"Line Number",
"Awardee",
"numberItem",
"DATE",
"AGENCY",
"OFFICE",
"LOCATION",
"ZIP",
"CLASSCOD",
"OFFADD",
"SUBJECT",
"SOLNBR",
"RESPDATE",
"CONTACT",
"DESC",
"URL",
"POPCOUNTRY",
"POPZIP",
"POPADDRESS",
"detailsSolicitation",
"ARCHDATE",
"EMAIL",
"SETASIDE",
"NTYPE",
"AWDNBR",
"AWDAMT",
"AWDDATE",
"AWARDEE",
"LINENBR",
"BR",
"Company Name",
"STAUTH",
"MODNBR",
"FOJA",
"DONBR",
"ADDRESS",
"CORRECTION",
"LINK",
"RECOVERY_ACT",
"DOCUMENT_PACKAGES",
"CHANGES",
"vendorisWomanOwnedBusiness",
"vendorisWomanOwnedBusinessSmallBusiness",
"agencySpecificIDVElementsagencyID",
"agencySpecificIDVElementsNASASpecificIDVElements",
"CONGRESSIONAL JUSTIFICATION",
"awardContractID.agencyID",
"awardContractID.PIID",
"awardContractID.modNumber",
"awardContractID.transactionNumber",
"referencedIDVID.agencyID",
"referencedIDVID.PIID",
"referencedIDVID.modNumber",
"signedDate",
"effectiveDate",
"currentCompletionDate",
"ultimateCompletionDate",
"obligatedAmount",
"baseAndExercisedOptionsValue",
"baseAndAllOptionsValue",
"contractingOfficeAgencyID",
"contractingOfficeID",
"fundingRequestingAgencyID",
"fundingRequestingOfficeID",
"foreignFunding",
"feePaidForUseOfService",
"contractActionType",
"typeOfContractPricing",
"reasonForModification",
"majorProgramCode",
"nationalInterestActionCode",
"descriptionOfContractRequirement",
"inherentlyGovernmentalFunction",
"GFE-GFP",
"undefinitizedAction",
"consolidatedContract",
"performanceBasedServiceContract",
"contingencyHumanitarianPeacekeepingOperation",
"treasuryAccount.agencyIdentifier",
"treasuryAccount.mainAccountCode",
"treasuryAccount.obligatedAmount",
"purchaseCardAsPaymentMethod",
"numberOfActions",
"ClingerCohenAct",
"materialsSuppliesArticlesEquipment",
"laborStandards",
"constructionWageRateRequirements",
"interagencyContractingAuthority",
"productOrServiceCode",
"contractBundling",
"principalNAICSCode",
"recoveredMaterialClauses",
"manufacturingOrganizationType",
"useOfEPADesignatedProducts",
"countryOfOrigin",
"placeOfManufacture",
"vendorName",
"isAlaskanNativeOwnedBusinessCorporationOrFirm",
"isAmericanIndianOwnedBusinessBusinessBusiness",
"isIndianTribe",
"isNativeHawaiianOwnedBusinessOrganizationOrFirm",
"isTriballyOwnedBusinessBusinessFirm",
"isVeteranOwnedBusiness",
"isServiceDisabledVeteranOwnedBusinessBusiness",
"isWomanOwnedBusiness",
"isMinorityOwnedBusiness",
"isSubcontinentAsianIndianAmericanOwnedBusiness",
"isAsianPacificAmericanOwnedBusiness",
"isBlackAmericanOwnedBusiness",
"isHispanicAmericanOwnedBusiness",
"isNativeAmericanOwnedBusiness",
"isOtherMinorityOwnedBusiness",
"isVerySmallBusiness",
"isWomanOwnedBusinessSmallBusiness",
"isEconomicallyDisadvantagedWomanOwnedSmallBusiness",
"isJointVentureWomenOwnedSmallBusiness",
"isJointVentureWomenOwnedSmallBusiness",
"isCommunityDevelopedCorporationOwnedFirm",
"isLaborSurplusAreaFirm",
"isUSFederalGovernment",
"isFederallyFundedResearchAndDevelopmentCorp",
"isFederalAgency",
"isStateGovernment",
"isLocalGovernment",
"isCityGovernment",
"isCountyGovernment",
"isInterMunicipalLocalGovernment",
"isLocalGovernment",
"isSchoolDistrictLocalGovernment",
"isSchoolDistrictLocalGovernment",
"isTownshipLocalGovernment",
"isTribalGovernment",
"isForeignGovernment",
"isCorporateEntityNonTaxExempt",
"isCorporateEntityTaxExempt",
"isPartnershipOrLLP",
"isSolePropreitorship",
"isSmallAgriculturalCooperative",
"isInternationalOrganization",
"isCommunityDevelopedCorporationOwnedFirm",
"isDomesticShelter",
"isEducationalInstitution",
"isFoundation",
"isHospital",
"isManufacturerOfGoods",
"isVeterinaryHospital",
"isHispanicServicingInstitution",
"receivesContracts",
"receivesGrants",
"receivesContractsAndGrants",
"isAirportAuthority",
"isCouncilOfGovernments",
"isHousingAuthoritiesPublicOrTribal",
"isInterstateEntity",
"isPlanningCommission",
"isPortAuthority",
"isTransitAuthority",
"isSubchapterSCorporation",
"isLimitedLiabilityCorporation",
"isForeignOwnedAndLocated",
"isForProfitOrganization",
"isNonprofitOrganization",
"isOtherNotForProfitOrganization",
"isShelteredWorkshop",
"stateOfIncorporation",
"countryOfIncorporation",
"organizationalType",
"is1862LandGrantCollege",
"is1890LandGrantCollege",
"is1994LandGrantCollege",
"isHistoricallyBlackCollegeOrUniversity",
"isMinorityInstitution",
"isPrivateUniversityOrCollege",
"isSchoolOfForestry",
"isStateControlledInstitutionOfHigherLearning",
"isTribalCollege",
"isVeterinaryCollege",
"isDOTCertifiedDisadvantagedBusinessEnterprise",
"isSelfCertifiedSmallDisadvantagedBusiness",
"isSBACertifiedSmallDisadvantagedBusiness",
"isSBACertified8AProgramParticipant",
"isSelfCertifiedHUBZoneJointVenture",
"isSBACertifiedHUBZone",
"isSBACertified8AJointVenture",
"vendorLocation.streetAddress",
"vendorLocation.city",
"vendorLocation.state",
"vendorLocation.ZIPCode",
"vendorLocation.countryCode",
"vendorLocation.phoneNo",
"vendorLocation.faxNo",
"vendorLocation.congressionalDistrictCode",
"vendorSiteCode",
"vendorAlternateSiteCode",
"vendorDUNSInformation.DUNSNumber",
"vendorDUNSInformation.cageCode",
"ccrRegistrationDetails.registrationDate",
"ccrRegistrationDetails.renewalDate",
"contractingOfficerBusinessSizeDetermination",
"principalPlaceOfPerformance.stateCode",
"principalPlaceOfPerformance.countryCode",
"placeOfPerformanceZIPCode",
"placeOfPerformanceCongressionalDistrict",
"extentCompeted",
"solicitationProcedures",
"typeOfSetAside",
"typeOfSetAsideSource",
"evaluatedPreference",
"numberOfOffersReceived",
"numberOfOffersSource",
"commercialItemAcquisitionProcedures",
"A76Action",
"localAreaSetAside",
"createdBy",
"createdDate",
"lastModifiedBy",
"lastModifiedDate",
"closedBy",
"closedDate",
"closedStatus",
"status",
"listOfAdditionalReportingValues.additionalReportingValue",
"vendorLegalOrganizationName",
"idvTypeOfSetAside",
"statutoryExceptionToFairOpportunity",
"idvNumberOfOffersReceived",
"commercialItemTestProgram",
"fedBizOpps",
"vendorAlternateName",
"costOrPricingData",
"multiYearContract",
"reasonNotCompeted",
"subcontractPlan",
"solicitationID",
"contractFinancing",
"vendorDoingAsBusinessName",
"informationTechnologyCommercialItemCategory",
"costAccountingStandardsClause",
"priceEvaluationPercentDifference",
"otherStatutoryAuthority",
"IDVID.agencyID",
"IDVID.PIID",
"IDVID.modNumber",
"lastDateToOrder",
"totalEstimatedOrderValue",
"whoCanUse",
"individualOrderLimit",
"typeOfIDC",
"multipleOrSingleAwardIDC",
"principalPlaceOfPerformance.locationCode",
"competitiveProcedures",
"smallBusinessCompetitivenessDemonstrationProgram",
"vendorLocation.streetAddress2",
"treasuryAccount.subAccountCode",
"CCRException",
"typeOfFeeForUseOfService",
"agencyID",
"modNumber",
"fiscalYear",
"nonGovernmentalDollars",
"typeOfAgreement",
"agencyIdentifier",
"mainAccountCode",
"nonTraditionalGovernmentContractorParticipation",
"PSCCode",
"isOtherbusinessOrOrganization",
"isUSGovernmentEntity",
"isArchitectureAndEngineering",
"isConstructionFirm",
"isResearchAndDevelopment",
"isServiceProvider",
"isAlaskanNativeServicingInstitution",
"isNativeHawaiianServicingInstitution",
"transactionNumber",
"subAccountCode",
"emailAddress",
"fixedFeeValue",
"orderingProcedure",
"Award Number",
"Project Title",
"Award Abstract",
"Anticipated Award Amount",
"DoD Awarding Office",
"Fiscal Year",
"Recipient Organization",
"PI First Name",
"PI Last Name",
"PI Type",
"Potential Period of Performance Start",
"Potential Period of Performance End",
"Creation Date",
"Modified Date",
"Contract Modification Number",
"IDV Type",
"Part 8 or Part 13",
"Multiple or Single Award IDC",
"Referenced IDV",
"Contracting Department",
"Contracting Department ID",
"PIID Agency",
"Agency ID",
"Contracting Office",
"Contracting Office ID",
"Last Date to Order",
"Ceiling Amount from IDV ($)",
"Dollars Obligated on Orders ($)",
"Ceiling Remaining ($)",
"nameNAICS",
"nameFullPSC",
"PSC Code",
"Website URL",
"POC Email",
"Requirement Description",
"Program Acronym",
"National Interest Action",
"Major Program",
"Fee for Use of Service",
"Max Order Limit ($)",
"Who Can Use",
"Contracting Officer's Business Size Determination",
"Socio Economic Data",
"tag",
"ACCT_NUM",
"ATA",
"AID",
"BPOA",
"EPOA",
"A",
"MAIN",
"SUB",
"GWA_TAS_NAME",
"Agency AID",
"FR Entity Type",
"FR Entity Description",
"financial_indicator_type2",
"FIN_IND_TYP2 description",
"Function Code",
"Function Description",
"Sub Function Code",
"Sub Function Description",
"DT_TM_ESTAB",
"DT_END",
"agencySpecificAwardElementsnonFederalFundingAmount",
"awardContractIDagencyID",
"awardContractIDPIID",
"awardContractIDmodNumber",
"awardContractIDtransactionNumber",
"referencedIDVIDagencyID",
"referencedIDVIDPIID",
"referencedIDVIDmodNumber",
"contractDataGFE-GFP",
"vendorHeadervendorName",
"vendorSocioEconomicIndicatorsisAlaskanNativeOwnedBusinessCorporationOrFirm",
"vendorSocioEconomicIndicatorsisAmericanIndianOwnedBusinessBusinessBusiness",
"vendorSocioEconomicIndicatorsisIndianTribe",
"vendorSocioEconomicIndicatorsisNativeHawaiianOwnedBusinessOrganizationOrFirm",
"vendorSocioEconomicIndicatorsisTriballyOwnedBusinessBusinessFirm",
"vendorSocioEconomicIndicatorsisSmallBusiness",
"vendorSocioEconomicIndicatorsisVeteranOwnedBusiness",
"vendorSocioEconomicIndicatorsisServiceDisabledVeteranOwnedBusinessBusiness",
"vendorSocioEconomicIndicatorsisWomanOwnedBusiness",
"minorityOwnedisMinorityOwnedBusiness",
"minorityOwnedisSubcontinentAsianIndianAmericanOwnedBusiness",
"minorityOwnedisAsianPacificAmericanOwnedBusiness",
"minorityOwnedisBlackAmericanOwnedBusiness",
"minorityOwnedisHispanicAmericanOwnedBusiness",
"minorityOwnedisNativeAmericanOwnedBusiness",
"minorityOwnedisOtherMinorityOwned",
"vendorSocioEconomicIndicatorsisVerySmallBusiness",
"vendorSocioEconomicIndicatorsisWomanOwnedBusinessSmallBusiness",
"vendorSocioEconomicIndicatorsisEconomicallyDisadvantagedWomanOwnedSmallBusiness",
"vendorSocioEconomicIndicatorsisJointVentureWomenOwnedSmallBusiness",
"vendorSocioEconomicIndicatorsisJointVentureWomenOwnedSmallBusiness",
"vendorBusinessTypesisCommunityDevelopedCorporationOwnedFirm",
"vendorBusinessTypesisLaborSurplusAreaFirm",
"federalGovernmentisFederalGovernment",
"federalGovernmentisFederallyFundedResearchAndDevelopmentCorp",
"federalGovernmentisFederalGovernmentAgency",
"vendorBusinessTypesisStateGovernment",
"localGovernmentisLocalGovernment",
"localGovernmentisCityLocalGovernment",
"localGovernmentisCountyLocalGovernment",
"localGovernmentisInterMunicipalLocalGovernment",
"localGovernmentisLocalGovernmentOwned",
"localGovernmentisMunicipalityLocalGovernment",
"localGovernmentisSchoolDistrictLocalGovernment",
"localGovernmentisTownshipLocalGovernment",
"vendorBusinessTypesisTribalGovernment",
"vendorBusinessTypesisForeignGovernment",
"businessOrOrganizationTypeisCorporateEntityNotTaxExempt",
"businessOrOrganizationTypeisCorporateEntityTaxExempt",
"businessOrOrganizationTypeisPartnershipOrLimitedLiabilityPartnership",
"businessOrOrganizationTypeisSolePropreitorship",
"businessOrOrganizationTypeisSmallAgriculturalCooperative",
"businessOrOrganizationTypeisInternationalOrganization",
"businessOrOrganizationTypeisUSGovernmentEntity",
"vendorLineOfBusinessisCommunityDevelopmentCorporation",
"vendorLineOfBusinessisDomesticShelter",
"vendorLineOfBusinessisEducationalInstitution",
"vendorLineOfBusinessisFoundation",
"vendorLineOfBusinessisHospital",
"vendorLineOfBusinessisManufacturerOfGoods",
"vendorLineOfBusinessisVeterinaryHospital",
"vendorLineOfBusinessisHispanicServicingInstitution",
"vendorRelationshipWithFederalGovernmentreceivesContracts",
"vendorRelationshipWithFederalGovernmentreceivesGrants",
"vendorRelationshipWithFederalGovernmentreceivesContractsAndGrants",
"typeOfGovernmentEntityisAirportAuthority",
"typeOfGovernmentEntityisCouncilOfGovernments",
"typeOfGovernmentEntityisHousingAuthoritiesPublicOrTribal",
"typeOfGovernmentEntityisInterstateEntity",
"typeOfGovernmentEntityisPlanningCommission",
"typeOfGovernmentEntityisPortAuthority",
"typeOfGovernmentEntityisTransitAuthority",
"vendorOrganizationFactorsisSubchapterSCorporation",
"vendorOrganizationFactorsisLimitedLiabilityCorporation",
"vendorOrganizationFactorsisForeignOwnedAndLocated",
"profitStructureisForProfitOrganization",
"profitStructureisNonprofitOrganization",
"profitStructureisOtherNotForProfitOrganization",
"vendorOrganizationFactorsisShelteredWorkshop",
"typeOfEducationalEntityis1862LandGrantCollege",
"typeOfEducationalEntityis1890LandGrantCollege",
"typeOfEducationalEntityis1994LandGrantCollege",
"typeOfEducationalEntityisHistoricallyBlackCollegeOrUniversity",
"typeOfEducationalEntityisMinorityInstitution",
"typeOfEducationalEntityisPrivateUniversityOrCollege",
"typeOfEducationalEntityisSchoolOfForestry",
"typeOfEducationalEntityisStateControlledInstitutionOfHigherLearning",
"typeOfEducationalEntityisTribalCollege",
"typeOfEducationalEntityisVeterinaryCollege",
"typeOfEducationalEntityisAlaskanNativeServicingInstitution",
"typeOfEducationalEntityisNativeHawaiianServicingInstitution",
"vendorCertificationsisDOTCertifiedDisadvantagedBusinessEnterprise",
"vendorCertificationsisSelfCertifiedSmallDisadvantagedBusiness",
"vendorCertificationsisSBACertifiedSmallDisadvantagedBusiness",
"vendorCertificationsisSBACertified8AProgramParticipant",
"vendorCertificationsisSelfCertifiedHUBZoneJointVenture",
"vendorCertificationsisSBACertifiedHUBZone",
"vendorCertificationsisSBACertified8AJointVenture",
"vendorLocationstreetAddress",
"vendorLocationcity",
"vendorLocationstate",
"vendorLocationZIPCode",
"vendorLocationcountryCode",
"vendorDUNSInformationDUNSNumber",
"vendorDUNSInformationvendorName",
"vendorDUNSInformationglobalParentDUNSNumber",
"vendorDUNSInformationglobalParentDUNSName",
"vendorcontractorName",
"vendorcontractingOfficerBusinessSizeDetermination",
"principalPlaceOfPerformancelocationCode",
"principalPlaceOfPerformancestateCode",
"principalPlaceOfPerformancecountryCode",
"treasuryAccountSymbolagencyIdentifier",
"treasuryAccountSymbolmainAccountCode",
"listOfAdditionalReportingValuesadditionalReportingValue",
"vendorOrganizationFactorsstateOfIncorporation",
"vendorOrganizationFactorscountryOfIncorporation",
"vendorOrganizationFactorsorganizationalType",
"vendorLocationphoneNo",
"vendorLocationfaxNo",
"vendorLocationcongressionalDistrictCode",
"vendorSiteDetailsvendorSiteCode",
"vendorSiteDetailsvendorAlternateSiteCode",
"vendorDUNSInformationcageCode",
"ccrRegistrationDetailsregistrationDate",
"ccrRegistrationDetailsrenewalDate",
"genericStringsgenericString01",
"treasuryAccountSymbolsubAccountCode",
"vendorHeadervendorAlternateName",
"vendorHeadervendorLegalOrganizationName",
"vendorHeadervendorDoingAsBusinessName",
"otherTransactionAwardContractIDagencyID",
"otherTransactionAwardContractIDPIID",
"otherTransactionAwardContractIDmodNumber",
"relevantContractDatesfiscalYear",
"dollarValuesnonGovernmentalDollars",
"totalDollarValuestotalNonGovernmentalDollars",
"contractDatatypeOfAgreement",
"contractDatanonTraditionalGovernmentContractorParticipation",
"contractDetailPSCCode",
"iDVIDagencyID",
"iDVIDPIID",
"iDVIDmodNumber",
"agencySpecificAwardElementsagencyID",
"pRNumber",
"closeoutPR",
"procurementPlacementCode",
"solicitationIssueDate",
"listOfContractAdministrationsDelegatedcontractAdministrationDelegated",
"advisoryOrAssistanceServicesContract",
"supportServicesTypeContract",
"newTechnologyOrPatentRightsClause",
"managementReportingRequirements",
"propertyFinancialReporting",
"valueEngineeringClause",
"securityCode",
"administratorCode",
"contractingOfficerCode",
"negotiatorCode",
"cOTRName",
"organizationCode",
"contractFundCode",
"isPhysicallyComplete",
"physicalCompletionDate",
"finalInvoicePaidDate",
"destroyDate",
"accountingInstallationNumber",
"principalInvestigatorfirstName",
"principalInvestigatorlastName",
"fieldOfScienceOrEngineering",
"installationUnique",
"offerorsProposalNumber",
"nonFederalFundingAmount",
"fundedThroughDate",
"cancellationDate",
"cFDAProgramIdentificationNumber",
"alternateCOTRName",
"vendorLocationstreetAddress2",
"alternatePrincipalInvestigatorfirstName",
"alternatePrincipalInvestigatorlastName",
"accessionNumber",
"principalInvestigatormiddleInitial",
"totalObligatedAmount",
"totalBaseAndExercisedOptionsValue",
"totalBaseAndAllOptionsValue",
"contractMarketingData",
"seaTransportation",
"treasuryAccountSymbol",
"claimantProgramCode",
"systemEquipmentCode",
"vendorHeader",
"isSmallBusiness",
"streetAddress",
"city",
"state",
"ZIPCode",
"countryCode",
"phoneNo",
"faxNo",
"congressionalDistrictCode",
"DUNSNumber",
"globalParentDUNSNumber",
"globalParentDUNSName",
"registrationDate",
"renewalDate",
"stateCode",
"preferencePrograms",
"approvedBy",
"approvedDate",
"referencedIDVPart8OrPart13",
"referencedIDVMultipleOrSingle",
"referencedIDVType",
"listOfAdditionalReportingValues",
"additionalReportingValue",
"cageCode",
"contractorName",
"contractID",
"websiteURL",
"locationCode",
"programAcronym",
"initiative",
"totalNonGovernmentalDollars",
"firstName",
"lastName",
"contractAdministrationDelegated",
"streetAddress2",
"middleInitial",
"agencySpecificIDVElements",
"NASASpecificIDVElements",
"isAlaskanNativeOwnedCorporationOrFirm",
"isAmericanIndianOwned",
"isNativeHawaiianOwnedOrganizationOrFirm",
"isTriballyOwnedFirm",
"isVeteranOwned",
"isServiceRelatedDisabledVeteranOwnedBusiness",
"isWomenOwned",
"isMinorityOwned",
"isSubContinentAsianAmericanOwnedBusiness",
"isOtherMinorityOwned",
"isWomenOwnedSmallBusiness",
"isEconomicallyDisadvantagedWomenOwnedSmallBusiness",
"isJointVentureEconomicallyDisadvantagedWomenOwnedSmallBusiness",
"isFederalGovernment",
"isFederalGovernmentAgency",
"isCityLocalGovernment",
"isCountyLocalGovernment",
"isLocalGovernmentOwned",
"isMunicipalityLocalGovernment",
"isCorporateEntityNotTaxExempt",
"isPartnershipOrLimitedLiabilityPartnership",
"isCommunityDevelopmentCorporation",
"modified",
"nameVendorDUNS",
"contractIDIDVIDagencyID",
"contractIDIDVIDPIID",
"contractIDIDVIDmodNumber",
"feeRangeLowerValue",
"feeRangeUpperValue",
"streetAddress3",
"divisionName",
"divisionNumberOrOfficeCode"
)
nameActual = c(
"idContractIDV",
"nameVendorParent",
"codeSubContractingPlan",
"typeSolicitationProcedure",
"typeAward",
"idAgencyAward",
"nameAgencyAward",
"idAward",
"idModAward",
"numberTransaction",
"datetimeContractSigned",
"datetimeContractEffective",
"datetimeContractCompletionCurrent",
"datetimeContractCompletionUltimate",
"amountObligation",
"amountBaseAndExercisedOption",
"amountBaseAllOption",
"amountObligationTotal",
"amountBaseAndExercisedOptionTotal",
"amountBaseAllOptionTotal",
"idAgencyPurchaser",
"nameAgencyPurchaser",
"idOfficePurchaser",
"nameOfficeAward",
"idAgencyFundingRequest",
"descriptionAgencyFundingRequest",
"idAgencyFunding",
"descriptionOfficeFundingRequest",
"codeDomesticForeignEntity",
"typeDomesticForeignEntity",
"amountMarketingDataFee",
"codeAction",
"typeAward",
"codeCostPricingData",
"typeCostPricingData",
"codeNationalInterestAction",
"nameNationalInterestAction",
"descriptionObligation",
"codeInherentlyGovernmentFunction",
"typeInherentlyGovernmentFunction",
"hasGFEOrGFP",
"typeGFEGFP",
"isSeaTransported",
"typeSeaTransportation",
"codeUndefinitizedAction",
"typeUndefinitizedAction",
"codeConsolidatedContract",
"typeConsolidatedContract",
"codePerformanceBasedServiceContract",
"typePerformanceBasedServiceContract",
"codeHumanitarianPeackeepingOperation",
"typeHumanitarianPeackeepingOperation",
"codeTreasuryAccounts",
"codeContractFinancing",
"typeContractFinancing",
"hasCreditCardPurchaseMethod",
"descriptionCreditCardPurchaseMethod",
"countActions",
"hasClingerCohenActPlanning",
"typeClingerCohenAct",
"codeMaterialsSuppliesArticlesEquipment",
"typeMaterialsSuppliesArticlesEquipment",
"hasLaborStandardsClause",
"typeLaborStandards",
"hasConstructionWageRequirements",
"typeConstructionWageRateRequirements",
"codeInterAgencyContractingAuthority",
"codeProductService",
"typeInterAgencyContractingAuthority",
"codeContractBundling",
"typeProductOrService",
"codeProgramClaimant",
"typeContractBundling",
"idNAICS",
"typeProgramClaimant",
"codeRecoveredMaterials",
"nameNAICS",
"codeManufacturingOrganization",
"typeRecoveredMaterials",
"codeEquipmentSystem",
"typeManufacturingOrganization",
"codeTechnologyItem",
"typeEquipmentSystem",
"codeEPADesignatedProduct",
"typeTechnologyItem",
"codeCountryOriginProductService",
"typeEPADesignatedProduct",
"codePlaceOfManufacture",
"countryProductServicePlaceOfManufacture",
"isAlaskanNativeOwnedBusiness",
"typeProductServicePlaceOfManufacture",
"isAmericanIndianOwnedBusinessBusinessBusiness",
"isIndianTribeOwned",
"isNativeHawaiianOwnedBusiness",
"isTriballyOwnedBusinessBusiness",
"isSmallBusiness",
"isVeteranOwnedBusiness",
"isServiceDisabledVeteranOwnedBusiness",
"isWomanOwnedBusiness",
"removeMinorityOwned",
"isVerySmallBusiness",
"isWomanOwnedBusinessSmallBusiness",
"isEconomicallyDisadvantagedWomanOwnedSmallBusiness",
"isJointVentureWomenOwnedSmallBusiness",
"isJointVentureWomenOwnedSmallBusiness",
"isCommunityDevelopedCorporationOwned",
"isLaborSurplusAreaFirm",
"removefederalGovernmentOwned",
"isStateGovernment",
"removeLocalGovernment",
"isTribalGovernment",
"isForeignGovernment",
"removeBusinessOrOrganizationType",
"isArchitectureAndEngineering",
"isCommunityDevelopedCorporationOwnedFirm",
"isConstructionFirm",
"isDomesticShelter",
"isEducationalInstitution",
"isFoundation",
"isHospital",
"isManufacturerOfGoods",
"isResearchAndDevelopment",
"isServiceProvider",
"isVeterinaryHospital",
"isHispanicServicingInstitution",
"hasContracts",
"hasGrants",
"hasContractsAndGrants",
"isAirportAuthority",
"isCouncilOfGovernments",
"isHousingAuthoritiesPublicOrTribal",
"isInterstateEntity",
"isPlanningCommission",
"isPortAuthority",
"isTransitAuthority",
"isSubchapterSCorporation",
"isLimitedLiabilityCorporation",
"isForeignOwnedAndLocated",
"removeVendorprofitStructure",
"isShelteredWorkshop",
"codeStateIncorporation",
"codeCountryIncorporation",
"nameStateIncorporation",
"typeOrganization",
"nameCountryIncorporation",
"is1862LandGrantCollege",
"is1890LandGrantCollege",
"is1994LandGrantCollege",
"isHistoricallyBlackCollegeOrUniversity",
"isMinorityInstitution",
"isPrivateUniversityOrCollege",
"isSchoolOfForestry",
"isStateControlledInstitutionOfHigherLearning",
"isTribalCollege",
"isVeterinaryCollege",
"isAlaskanNativeServicingInstitution",
"isNativeHawaiianServicingInstitution",
"isDOTCertifiedDisadvantagedBusinessEnterprise",
"isSelfCertifiedSmallDisadvantagedBusiness",
"isSBACertifiedSmallDisadvantagedBusiness",
"isSBACertified8AProgramParticipant",
"isSelfCertifiedHUBZoneJointVenture",
"isSBACertifiedHUBZone",
"isSBACertified8AJointVenture",
"addressStreet1Vendor",
"cityVendor",
"codeStateVendor",
"zipcodeVendor",
"removeBendorstateDescription",
"codeCountryVendor",
"zipcodeAreaVendor",
"telephoneVendor",
"removeVendorcountryCodeDescription",
"faxVendor",
"codeCongressionalDistrictVendor",
"remove_vendortext",
"slugDUNS",
"nameVendorDUNS",
"slugDUNSParent",
"nameVendorParent",
"datetimeRegistrationSAM",
"datetimeRenewalSAM",
"codeStatePerformance",
"codeCountryPerformance",
"zipcodePerformance",
"codeCongressionalDistrictPerformance",
"codeContractCompetition",
"codeSolicitationProcedures",
"typeContractCompetition",
"codeSetAside",
"typeSolicitationProcedures",
"codeEvaluatedPreference",
"typeSetAside",
"codeResearch",
"typeEvaluatedPreference",
"countOffersReceived",
"typeResearch",
"codeCommercialAcquisitionProcedures",
"isCommercialItemTestProgram",
"typeCommercialAcquisitionProcedures",
"hasA76FairActAction",
"typeCommercialItemTestProgram",
"isFBOSourced",
"typeA76Action",
"isLocalAreaSetAside",
"typeFBO",
"codeSubContractPlan",
"typeLocalAreaSetAside",
"transactionCreatedBy",
"typeSubContractPlan",
"datetimeTransactionCreated",
"transactionLastModifiedBy",
"datetimeTransactionLastModified",
"codeTransactionStatus",
"transactionApprovedBy",
"typeInformationStatus",
"datetimeTransactionApproved",
"isMinorityOwnedBusiness",
"isSubcontinentAsianIndianAmericanOwnedBusiness",
"isAsianPacificAmericanOwnedBusiness",
"isBlackAmericanOwnedBusiness",
"isHispanicAmericanOwnedBusiness",
"isNativeAmericanOwnedBusiness",
"isOtherMinorityOwnedBusiness",
"isUSFederalGovernment",
"isFederallyFundedResearchAndDevelopmentCorp",
"isFederalAgency",
"isLocalGovernment",
"isCityGovernment",
"isCountyGovernment",
"isInterMunicipalLocalGovernment",
"isLocalGovernment",
"isSchoolDistrictLocalGovernment",
"isSchoolDistrictLocalGovernment",
"isTownshipLocalGovernment",
"isCorporateEntityNonTaxExempt",
"isCorporateEntityTaxExempt",
"isPartnershipOrLLP",
"isSolePropreitorship",
"isSmallAgriculturalCooperative",
"isInternationalOrganization",
"isOtherBusiness",
"isUSGovernmentEntity",
"isForProfitOrganization",
"isNonprofitOrganization",
"isOtherNotForProfitOrganization",
"nameContract",
"urlFPDSContractAtom",
"datetimeContractModified",
"codeContractModification",
"typeContractModification",
"hasCostOrPricingData",
"typeCostOrPricing",
"codeCostAccountingClause",
"typeCostAccountingClause",
"isMultiYearContract",
"typeContractMultiYears",
"codePerformanceLocation",
"isSmallBusinessCompetitivenessDemonstrationProgram",
"pctPriceDifferencePriceEvaluation",
"idSolicitation",
"typeAdditionalReportingValues",
"cageVendor",
"codeSetAsideSource",
"typeSetAsideSource",
"codeOffersSource",
"typeOffersSource",
"isTransactionClosed",
"idAgencyAward",
"nameAgencyAward",
"idContract",
"codeModification",
"datetimeLastContractOrder",
"amountEstimatedTotal",
"urlGovernmentData",
"codeWhoCanUse",
"emailContract",
"amountIndividualOrderLimit",
"codeIDC",
"typeIDC",
"codeStatutoryAuthority",
"codeIDVMultipleOrSingle",
"typeAwardMultipleOrSingle",
"codeIDV",
"typeAward",
"hasFeeForUse",
"typeFeeForUse",
"nameMajorProgram",
"codeIDCContract",
"typeIDCContract",
"acronymProgram",
"amountFeeLowerBound",
"amountFeeUpperBound",
"typeOrderingProcedure",
"codeFairOpportunityException",
"typeFairOpportunityException",
"transactionClosedBy",
"datetimeTransactionClosed",
"codeNotCompeteReason",
"typeNotCompeteReason",
"codeIDVPart8OrPart13",
"addressStreet2Vendor",
"codeIDVSetAside",
"typeAwardSetAside",
"countOffersReceivedIDV",
"datetimeGenericStrings",
"idAgencyAwardSpecific",
"idPRNumber",
"hasPRCloseOut",
"codePlacementAgency",
"isAdvisoryOrAssistanceServicesContract",
"hasSupportServicesContract",
"hasNewTechnologyOrPatentRightsClause",
"hasManagementReportingRequirements",
"hasPropertyFinancialReporting",
"hasEngineeringClause",
"hasSecurityCode",
"codeAdministrator",
"codeContractingOfficer",
"codeNegotiator",
"nameCOTR",
"codeAwardOrganization",
"codeContractFund",
"isPhysicallyComplete",
"idAccountingInstallation",
"datetimeFundedThroughDate",
"codePrincipalInvestigator",
"idScienceOrEngineering",
"datetimeCompletionPhysical",
"idAccession",
"datetimeDestroy",
"datetimeSolicitationIssued",
"codeContractAdministrationsDelegated",
"datetimeFinalInvoicePaid",
"amountMarketingDataFeeFixed",
"isWalshHealyAct",
"removeWalshHealyActDescription",
"isMandatedServiceContractAct",
"removeMandatesserviceContractActDescription",
"isDavisBaconAct",
"removeMandatesDavisBaconActDescription",
"idContract",
"idContractIDV",
"idAgency",
"codeModification",
"idAgencyAward",
"nameAgencyAward",
"idOfficeAward",
"nameOfficeAward",
"idAgencyFunding",
"idOfficeFunding",
"nameOfficeFunding",
"codeAgency",
"nameAgency",
"idDepartment",
"nameDepartment",
"dateLastModified",
"lastModifiedBy",
"dateAwardCompletion",
"dateCreated",
"dateSigned",
"dateEffective",
"dateCompletion",
"amountObligation",
"amountContract",
"typeCostPricingData",
"statusAward",
"typeContract",
"createdBy",
"descriptionRequirement",
"modificationReason",
"legislativeMandates",
"localAreaSetAside",
"indicatorsSocioEconomic",
"isMultiYearContract",
"codeNationalInterestAction",
"nameNationalInterestAction",
"idNAICS",
"nameNAICS",
"codeProductService",
"descriptionProductService",
"codeCongressionalDistrictPerformance",
"countryPerformance",
"nameStatePerformance",
"cityVendor",
"codeCongressionalDistrictVendor",
"codeCountryVendor",
"nameCountryVendor",
"idDUNSVendor",
"nameVendorDoingBusinessAs",
"nameVendor",
"codeStateVendor",
"nameStateVendor",
"zipcodeVendor",
"nameVendorGlobal",
"codeTreasuryAccountSymbol",
"codeInstallnentAward",
"idContract",
"idContractIDV",
"codeModification",
"numberTransaction",
"typeAward",
"amountObligation",
"dateContractSigned",
"dateSolicitation",
"idAgencyAward",
"nameAgencyAward",
"nameOfficeAward",
"typeProductOrService",
"codeProductService",
"descriptionProductService",
"idNAICS",
"nameNAICS",
"slugDUNS",
"nameVendor",
"cityVendor",
"codeStateVendor",
"zipcodeVendor",
"slugDUNSParent",
"nameVendorParent",
"codeAdditionalReporting",
"descriptionAdditonalReporting",
"idAgency",
"idDepartment",
"nameAgency",
"slugAgency",
"isBrokerRegistered",
"isASPRegistered",
"sourceOffice",
"idOffice",
"nameOffice",
"slugOffice",
"nameAdminister",
"idAdminister",
"codeFREC",
"descriptionFREC",
"sourceOfficeCode",
"isFPDSOffice",
"isASPRegisteredOld",
"nameAgencyOld",
"nameOfficeOld",
"isFREC",
"descriptionMission",
"url",
"icon",
"idAgency",
"nameAgency",
"idBureau",
"nameBureau",
"idAccount",
"nameAccount",
"idTreasuryAccount",
"idFunction",
"nameFunction",
"typeCategoryBEA",
"typeBudget",
"fy1976",
"fyTQ",
"fy1977",
"fy1978",
"fy1979",
"fy1980",
"fy1981",
"fy1982",
"fy1983",
"fy1984",
"fy1985",
"fy1986",
"fy1987",
"fy1988",
"fy1989",
"fy1990",
"fy1991",
"fy1992",
"fy1993",
"fy1994",
"fy1995",
"fy1996",
"fy1997",
"fy1998",
"fy1999",
"fy2000",
"fy2001",
"fy2002",
"fy2003",
"fy2004",
"fy2005",
"fy2006",
"fy2007",
"fy2008",
"fy2009",
"fy2010",
"fy2011",
"fy2012",
"fy2013",
"fy2014",
"fy2015",
"fy2016",
"fy2017",
"fy2018",
"fy2019",
"fy2020",
"fy2021",
"fy2022",
"nameCountryPerformance",
"idDUNSParent",
"codeResearch",
"nameAgency",
"idAgency",
"typeDescriptionManufacter",
"codeCountryOriginProductService",
"idContract",
"dateCompletion",
"idNAICS",
"codeProductService",
"descriptionProductService",
"countryVendor",
"idDUNSVendor",
"typeContract",
"typeAward",
"amountContract",
"amountObligation",
"nameSolicitation",
"idSolicitation",
"nameAgency",
"nameOffice",
"detailsOffice",
"typeSolicitation",
"typeSolicitationOriginal",
"datePosted",
"datePostedOriginal",
"descriptionDeadLine",
"policyArchiving",
"datetimeArchive",
"typeSetAside",
"codeClassification",
"idNAICS",
"addressContractingOffice",
"descriptionContractingPointContact",
"locationContractPerformance",
"detailsAdditionalLinks",
"descriptionSolicitation",
"isActive",
"urlSolicitation",
"idAward",
"datePosted",
"dateAward",
"amountAward",
"descriptionLineNumber",
"nameAwardeeAddress",
"numberItem",
"dateSolicitation",
"nameAgency",
"nameOffice",
"detailsOffice",
"zipcodeAgency",
"codeClassification",
"locationOffice",
"nameSolicitation",
"idSolicitation",
"dateResponse",
"descriptionContractingPointContact",
"descriptionSolicitation",
"urlSolicitation",
"codeCountryPerformance",
"zipcodePerformace",
"locationContractPerformance",
"detailsSolicitation",
"dateArchive",
"emailContact",
"typeSetAside",
"typeAward",
"idAward",
"amountAward",
"dateAward",
"nameAwardeeAddress",
"descriptionLineNumber",
"requirementsBureau",
"nameCompany",
"typeFAR",
"codeModification",
"descriptionFollowOn",
"idDONBR",
"emailContact",
"hasCorrections",
"urlSolicitation",
"isRecoveryAct",
"descriptionDocumentPackages",
"descriptionChanges",
"isWomanOwnedBusiness",
"isWomanOwnedBusinessSmallBusiness",
"idAgency",
"idContractIDV",
"urlCongressionalJustification",
"idAgencyAward",
"idContract",
"codeModification",
"numberTransaction",
"idOfficeIDV",
"idContractIDV",
"codeModIDV",
"datetimeContractSigned",
"datetimeContractEffective",
"datetimeContractCompletionCurrent",
"datetimeContractCompletionUltimate",
"amountObligation",
"amountBaseAndExercisedOption",
"amountBaseAllOption",
"idAgencyAward",
"idOfficeAward",
"idAgencyFunding",
"idOfficeFunding",
"codeDomesticForeignEntity",
"amountFeePaidForService",
"codeAction",
"codeCostPricingData",
"codeContractModification",
"nameMajorProgram",
"codeNationalInterestAction",
"descriptionObligation",
"codeInherentlyGovernmentFunction",
"hasGFEOrGFP",
"codeUndefinitizedAction",
"codeConsolidatedContract",
"codePerformanceBasedServiceContract",
"codeHumanitarianPeackeepingOperation",
"idCGAC",
"codeAccountOMB",
"amountObligationTreasury",
"hasCreditCardPurchaseMethod",
"countActions",
"hasClingerCohenActPlanning",
"codeMaterialsSuppliesArticlesEquipment",
"hasLaborStandardsClause",
"hasConstructionWageRequirements",
"codeInterAgencyContractingAuthority",
"codeProductService",
"codeContractBundling",
"idNAICS",
"codeRecoveredMaterials",
"codeManufacturingOrganization",
"codeEPADesignatedProduct",
"codeCountryOriginProductService",
"codePlaceOfManufacture",
"nameVendor",
"isAlaskanNativeOwnedBusinessCorporationOrFirm",
"isAmericanIndianOwnedBusinessBusinessBusiness",
"isIndianTribeOwned",
"isNativeHawaiianOwnedBusiness",
"isTriballyOwnedBusinessBusiness",
"isVeteranOwnedBusiness",
"isServiceDisabledVeteranOwnedBusiness",
"isWomanOwnedBusiness",
"isMinorityOwnedBusiness",
"isSubcontinentAsianIndianAmericanOwnedBusiness",
"isAsianPacificAmericanOwnedBusiness",
"isBlackAmericanOwnedBusiness",
"isHispanicAmericanOwnedBusiness",
"isNativeAmericanOwnedBusiness",
"isOtherMinorityOwnedBusiness",
"isVerySmallBusiness",
"isWomanOwnedSmallBusiness",
"isEconomicallyDisadvantagedWomanOwnedSmallBusiness",
"isJointVentureWomenOwnedSmallBusiness",
"isJointVentureWomenOwnedSmallBusiness",
"isCommunityDevelopedCorporationOwned",
"isLaborSurplusAreaFirm",
"isUSFederalGovernment",
"isFederallyFundedResearchAndDevelopmentCorp",
"isFederalAgency",
"isStateGovernment",
"isLocalGovernment",
"isCityGovernment",
"isCountyGovernment",
"isInterMunicipalLocalGovernment",
"isLocalGovernment",
"isSchoolDistrictLocalGovernment",
"isSchoolDistrictLocalGovernment",
"isTownshipLocalGovernment",
"isTribalGovernment",
"isForeignGovernment",
"isCorporateEntityNonTaxExempt",
"isCorporateEntityTaxExempt",
"isPartnershipOrLLP",
"isSolePropreitorship",
"isSmallAgriculturalCooperative",
"isInternationalOrganization",
"isCommunityDevelopedCorporationOwnedFirm",
"isDomesticShelter",
"isEducationalInstitution",
"isFoundation",
"isHospital",
"isManufacturerOfGoods",
"isVeterinaryHospital",
"isHispanicServicingInstitution",
"hasContracts",
"hasGrants",
"hasContractsAndGrants",
"isAirportAuthority",
"isCouncilOfGovernments",
"isHousingAuthoritiesPublicOrTribal",
"isInterstateEntity",
"isPlanningCommission",
"isPortAuthority",
"isTransitAuthority",
"isSubchapterSCorporation",
"isLimitedLiabilityCorporation",
"isForeignOwnedAndLocated",
"isForProfitOrganization",
"isNonprofitOrganization",
"isOtherNotForProfitOrganization",
"isShelteredWorkshop",
"codeStateIncorporation",
"codeCountryIncorporation",
"typeOrganization",
"is1862LandGrantCollege",
"is1890LandGrantCollege",
"is1994LandGrantCollege",
"isHistoricallyBlackCollegeOrUniversity",
"isMinorityInstitution",
"isPrivateUniversityOrCollege",
"isSchoolOfForestry",
"isStateControlledInstitutionOfHigherLearning",
"isTribalCollege",
"isVeterinaryCollege",
"isDOTCertifiedDisadvantagedBusinessEnterprise",
"isSelfCertifiedSmallDisadvantagedBusiness",
"isSBACertifiedSmallDisadvantagedBusiness",
"isSBACertified8AProgramParticipant",
"isSelfCertifiedHUBZoneJointVenture",
"isSBACertifiedHUBZone",
"isSBACertified8AJointVenture",
"addressStreet1Vendor",
"cityVendor",
"codeStateVendor",
"zipcodeVendor",
"codeCountryVendor",
"telephoneVendor",
"faxVendor",
"codeCongressionalDistrictVendor",
"codeSiteVendor",
"codeSiteAlternateVendor",
"slugDUNS",
"cageVendor",
"datetimeRegistrationSAM",
"datetimeRenewalSAM",
"codeContractingOfficerBusinessSizeDetermination",
"codeStatePerformance",
"codeCountryPerformance",
"zipcodePerformance",
"codeCongressionalDistrictPerformance",
"codeContractCompetition",
"codeSolicitationProcedures",
"codeSetAside",
"codeSetAsideSource",
"codeEvaluatedPreference",
"countOffersReceived",
"codeOffersSource",
"codeCommercialAcquisitionProcedures",
"hasA76FairActAction",
"isLocalAreaSetAside",
"transactionCreatedBy",
"datetimeTransactionCreated",
"transactionLastModifiedBy",
"datetimeTransactionLastModified",
"transactionClosedBy",
"datetimeTransactionClosed",
"isTransactionClosed",
"codeStatusTransaction",
"codeAdditionalReporting",
"nameVendorLegal",
"codeIDVSetAside",
"codeFairOpportunityException",
"countOffersReceivedIDV",
"isCommercialItemTestProgram",
"isFBOSourced",
"nameVendorAlternate",
"hasCostOrPricingData",
"isMultiYearContract",
"codeNotCompeteReason",
"codeSubContractPlan",
"idSolicitation",
"codeContractFinancing",
"nameVendorDoingBusinessAs",
"codeTechnologyItem",
"codeCostAccountingClause",
"pctPriceDifferencePriceEvaluation",
"codeStatutoryAuthority",
"idAgencyAward",
"idContract",
"codeModification",
"dateLastOrder",
"amountEstimatedTotal",
"codeWhoCanUse",
"amountIndividualOrderLimit",
"codeIDCContract",
"codeIDC",
"codePerformanceLocation",
"codeCompetitiveProcedures",
"hasSmallBusinessCompetitivenessDemonstrationProgram",
"addressStreet2Vendor",
"codeTreasurySubAccountSymbol",
"idCCRException",
"codeFeeForUseOfService",
"idAgencyAward",
"codeModification",
"yearFiscalContract",
"amountNonGovernmentalDollars",
"typeAgreement",
"idCGAC",
"codeAccountOMB",
"codeNonTraditionalGovernmentContractorParticipation",
"codeProductService",
"isOtherBusiness",
"isUSGovernmentEntity",
"isArchitectureAndEngineering",
"isConstructionFirm",
"isResearchAndDevelopment",
"isServiceProvider",
"isAlaskanNativeServicingInstitution",
"isNativeHawaiianServicingInstitution",
"numberTransaction",
"codeTreasurySubAccountSymbol",
"emailContract",
"amountMarketingDataFeeFixed",
"typeOrderingProcedure",
"idAward",
"nameAward",
"descriptionAward",
"amountAward",
"nameOffice",
"yearFiscal",
"nameOrganization",
"nameFirstPrincipalInvestigator",
"nameLastPrincipalInvestigator",
"typePrincipalInvestigator",
"dateAwardStart",
"dateAwardEnd",
"datetimeCreated",
"datetimeModified",
"idModification",
"typeIDV",
"typePart8Part13",
"typeIDC",
"idContractIDV",
"nameDepartment",
"idDepartment",
"nameAgencyPIID",
"idAgency",
"nameOfficeAward",
"codeOfficePurchaser",
"dateOrderLast",
"amountCeilingIDV",
"amountObligation",
"amountRemaining",
"nameNAICS",
"nameFullPSC",
"codePSC",
"urlProgram",
"emailContact",
"descriptionRequirement",
"codeProgram",
"codeNationalInterestAction",
"nameMajorProgram",
"typeFeeForService",
"amountOrderMaximum",
"descriptionWhoCanUse",
"typeContractingOfficerBusinessSize",
"descriptionSetAsides",
"tag",
"idAccount",
"idATA",
"idAID",
"idBPOA",
"idEPOA",
"slugA",
"idMainAccount",
"idSubAccount",
"nameAccount",
"idAgency",
"slugOffice",
"nameOffice",
"slugFinancialIndicator",
"nameFinancialIndicator",
"idFunction",
"descriptionFunction",
"idSubFunction",
"descriptionSubFunction",
"datetimeEstablished",
"datetimeEnd",
"hasFundingNonFederal",
"idAgency",
"idContract",
"codeModification",
"transactionNumber",
"idAgencyIDV",
"idContractIDV",
"codeModIDV",
"hasGFEOrGFP",
"nameVendor",
"isAlaskanNativeOwnedBusinessCorporationOrFirm",
"isAmericanIndianOwnedBusinessBusinessBusiness",
"isIndianTribe",
"isNativeHawaiianOwnedBusinessOrganizationOrFirm",
"isTriballyOwnedBusinessBusinessFirm",
"isSmallBusiness",
"isVeteranOwnedBusiness",
"isServiceDisabledVeteranOwnedBusinessBusiness",
"isWomanOwnedBusiness",
"isMinorityOwnedBusiness",
"isSubcontinentAsianIndianAmericanOwnedBusiness",
"isAsianPacificAmericanOwnedBusiness",
"isBlackAmericanOwnedBusiness",
"isHispanicAmericanOwnedBusiness",
"isNativeAmericanOwnedBusiness",
"isOtherMinorityOwnedBusiness",
"isVerySmallBusiness",
"isWomanOwnedBusinessSmallBusiness",
"isEconomicallyDisadvantagedWomanOwnedSmallBusiness",
"isJointVentureWomenOwnedSmallBusiness",
"isJointVentureWomenOwnedSmallBusiness",
"isCommunityDevelopedCorporationOwnedFirm",
"isLaborSurplusAreaFirm",
"isUSFederalGovernment",
"isFederallyFundedResearchAndDevelopmentCorp",
"isFederalAgency",
"isStateGovernment",
"isLocalGovernment",
"isCityGovernment",
"isCountyGovernment",
"isInterMunicipalLocalGovernment",
"isLocalGovernment",
"isSchoolDistrictLocalGovernment",
"isSchoolDistrictLocalGovernment",
"isTownshipLocalGovernment",
"isTribalGovernment",
"isForeignGovernment",
"isCorporateEntityNonTaxExempt",
"isCorporateEntityTaxExempt",
"isPartnershipOrLLP",
"isSolePropreitorship",
"isSmallAgriculturalCooperative",
"isInternationalOrganization",
"isUSGovernmentEntity",
"isCommunityDevelopedCorporationOwnedFirm",
"isDomesticShelter",
"isEducationalInstitution",
"isFoundation",
"isHospital",
"isManufacturerOfGoods",
"isVeterinaryHospital",
"isHispanicServicingInstitution",
"hasFederalContracts",
"hasFederalGrants",
"hasFederalContractsAndGrants",
"isAirportAuthority",
"isCouncilOfGovernments",
"isHousingAuthoritiesPublicOrTribal",
"isInterstateEntity",
"isPlanningCommission",
"isPortAuthority",
"isTransitAuthority",
"isSubchapterSCorporation",
"isLimitedLiabilityCorporation",
"isForeignOwnedAndLocated",
"isForProfitOrganization",
"isNonprofitOrganization",
"isOtherNotForProfitOrganization",
"isShelteredWorkshop",
"is1862LandGrantCollege",
"is1890LandGrantCollege",
"is1994LandGrantCollege",
"isHistoricallyBlackCollegeOrUniversity",
"isMinorityInstitution",
"isPrivateUniversityOrCollege",
"isSchoolOfForestry",
"isStateControlledInstitutionOfHigherLearning",
"isTribalCollege",
"isVeterinaryCollege",
"isAlaskanNativeServicingInstitution",
"isNativeHawaiianServicingInstitution",
"isDOTCertifiedDisadvantagedBusinessEnterprise",
"isSelfCertifiedSmallDisadvantagedBusiness",
"isSBACertifiedSmallDisadvantagedBusiness",
"isSBACertified8AProgramParticipant",
"isSelfCertifiedHUBZoneJointVenture",
"isSBACertifiedHUBZone",
"isSBACertified8AJointVenture",
"vendorLocationstreetAddress",
"vendorLocationcity",
"vendorLocationstate",
"vendorLocationZIPCode",
"vendorLocationcountryCode",
"slugDUNS",
"nameVendor",
"slugDUNSParent",
"nameVendorParent",
"nameContractorVendor",
"codeContractingOfficerBusinessSizeDetermination",
"codePerformanceLocation",
"codeStatePerformance",
"codeCountryPerformance",
"idCGACAccount",
"codeAccountTreasury",
"codeAdditionalReporting",
"codeStateIncorporation",
"codeCountryIncorporation",
"typeOrganization",
"telephoneVendor",
"faxVendor",
"codeCongressionalDistrictVendor",
"codeSiteVendor",
"codeSiteAlternateVendor",
"cageVendor",
"datetimeRegistrationSAM",
"datetimeRenewalSAM",
"datetimeGeneric",
"codeSubAccountTreasury",
"nameVendorAlternate",
"nameVendorLegal",
"nameVendorDBA",
"idAgencyOtherTransactions",
"idContractOtherTransactions",
"codeModOtherTransactions",
"yearFiscalContract",
"amountObligationNonGovernment",
"amountTotalObligationNonGovernment",
"typeAgreement",
"codeNonTraditionalParticipation",
"codeProductService",
"idAgencyIDV",
"idContractIDV",
"codeModIDV",
"idAgency",
"numberPR",
"isCloseoutPR",
"codeProcurementPlacement",
"datetimeSolicitation",
"codeContractAdministrationDelegation",
"isAdvisoryOrAssistanceServicesContract",
"isSupportServicesContract",
"hasNewTechnologyOrPatentRightsClause",
"hasManagementReportingRequirements",
"hasPropertyFinancialReporting",
"hasValueEngineeringClause",
"hasSecurityCode",
"codeAdministrator",
"codeContractingOfficer",
"codeNegotiator",
"nameCOTRSignatory",
"codeOrganization",
"codeContractFund",
"isPhysicallyComplete",
"datetimePhysicalCompletion",
"datetimeFinalInvoicePaid",
"datetimeDestroy",
"numberAccountingInstallation",
"nameFirstPrincipalInvestigator",
"nameLastPrincipalInvestigator",
"numberFieldOfScienceOrEngineering",
"codeInstallationUnique",
"numberOfferorsProposal",
"amountFundingNonFederal",
"datetimeFundedThrough",
"datetimeCanceled",
"codeProgramFDA",
"nameCOTRAlternate",
"vendorLocationstreetAddress2",
"nameFirstPrincipalInvestigatorAlternate",
"nameLastPrincipalInvestigatorAlternate",
"idAccession",
"nameMiddlePrincipalInvestigator",
"amountObligationTotal",
"amountBaseAndExercisedOptionTotal",
"amountBaseAllOptionTotal",
"amountMarketingDataFee",
"isSeaTransported",
"codeTreasuryAccountSymbol",
"codeProgramClaimant",
"codeEquipmentSystem",
"nameVendorHeader",
"isSmallBusiness",
"addressStreet1",
"city",
"codeState",
"zipcode",
"codeCountry",
"telephone",
"fax",
"codeCongressionalDistrict",
"slugDUNS",
"slugDUNSParent",
"nameVendorParent",
"datetimeRegistrationSAM",
"datetimeRenewalSAM",
"codeState",
"codeSubContractPlan",
"transactionApprovedBy",
"datetimeTransactionApproved",
"codePart8OrPart13IDV",
"codeMultipleOrSingleIDV",
"codeReferencedIDV",
"codeListOfAdditionalReportingValues",
"codeAdditionalReportingValue",
"cageVendor",
"nameContractor",
"idContract",
"urlWebsite",
"location",
"codeProgram",
"idInitiative",
"amountNonGovernmentalDollars",
"nameFirst",
"nameLast",
"codeContractAdministrationDelegated",
"addressStreet2",
"initialMiddle",
"codeIDVElementsAgencySpecific",
"codeIDVElementsNASA",
"isAlaskanNativeOwnedCorporationOrFirm",
"isAmericanIndianOwned",
"isNativeHawaiianOwnedOrganizationOrFirm",
"isTriballyOwnedFirm",
"isVeteranOwned",
"isServiceRelatedDisabledVeteranOwnedBusiness",
"isWomenOwned",
"isMinorityOwned",
"isSubContinentAsianAmericanOwnedBusiness",
"isOtherMinorityOwned",
"isWomenOwnedSmallBusiness",
"isEconomicallyDisadvantagedWomenOwnedSmallBusiness",
"isJointVentureEconomicallyDisadvantagedWomenOwnedSmallBusiness",
"isFederalGovernment",
"isFederalGovernmentAgency",
"isCityLocalGovernment",
"isCountyLocalGovernment",
"isLocalGovernmentOwned",
"isMunicipalityLocalGovernment",
"isCorporateEntityNotTaxExempt",
"isPartnershipOrLimitedLiabilityPartnership",
"isCommunityDevelopmentCorporation",
"datetimeModified",
"nameVendorDUNS",
"idAgencyIDV2",
"idContractIDV2",
"codeModIDV2",
"amountfeeRangeLowerValue",
"amountfeeRangeUpperValue",
"addressStreet3",
"nameDivision",
"codeDivision"
)
tibble(nameFPDS, nameActual)
})
# munge -------------------------------------------------------------------
.pad_fdps_duns <-
function(duns = 3457,
total_characters = 9,
return_only_slug = T) {
zeros <- total_characters - nchar(duns)
if (zeros == 0) {
return(tibble(idDUNS = duns, slugDUNS = as.character(duns)))
}
zero_slug <- rep(0, zeros) %>% str_c(collapse = "")
slugDUNS <- glue("{zero_slug}{duns}") %>% as.character()
if (return_only_slug) {
return(slugDUNS)
}
tibble(idDUNS = duns, slugDUNS)
}
#' Pad DUNS for FDPS search
#'
#' @param duns vector of DUNS
#'
#' @return
#' @export
#'
#' @examples
pad_fdps_duns <-
function(duns = NULL) {
if (length(duns) == 0) {
stop("Enter DUNS")
}
duns %>%
map_dfr(function(duns) {
.pad_fdps_duns(duns = duns, return_only_slug = F)
})
}
.munge_fdps_data <-
function(data) {
data %>% .munge_data(clean_address = F)
}
.munge_fpds_names <-
function(data) {
dict_names <- dictionary_fpds_names()
fdps_names <-
names(data)
actual_names <-
fdps_names %>%
map_chr(function(x) {
df_row <-
dict_names %>%
mutate(nameFPDS = str_to_lower(nameFPDS)) %>%
filter(nameFPDS == str_to_lower(x)) %>%
distinct() %>%
slice(1)
if (nrow(df_row) == 0) {
glue::glue("Missing {x}") %>% message()
return(x)
}
df_row$nameActual
})
data %>%
set_names(actual_names)
}
.sort_slug <-
function(sort_method = NULL,
sort_descending = T) {
if (length(sort_method) == 0) {
return("")
}
sort_dir <- case_when(sort_descending ~ "desc=Y",
TRUE ~ "desc=N")
sort <-
sort_method %>% str_to_lower()
sort_slug <-
case_when(
sort %>% str_detect("amount") ~ glue("&sortBy=OBLIGATED_AMOUNT&{sort_dir}") %>% as.character(),
sort %>% str_detect("agency code") ~ glue("&sortBy=AGENCY_CODE&{sort_dir}") %>% as.character(),
sort %>% str_detect("agency name") ~ glue("sortBy=AGENCY_FULL_NAME&{sort_dir}") %>% as.character(),
sort %>% str_detect("date") ~ glue("sortBy=SIGNED_DATE&{sort_dir}") %>% as.character(),
sort %>% str_detect("contracing agency id") ~ glue("sortBy=CONTRACTING_AGENCY_ID&{sort_dir}") %>% as.character(),
sort %>% str_detect(
"contract agency name|contracting agency name|contracting agency full name"
) ~ glue("sortBy=CONTRACTING_AGENCY_NAME&{sort_dir}") %>% as.character(),
sort %>% str_detect("department name") ~ glue("sortBy=DEPARTMENT_FULL_NAME&{sort_dir}") %>% as.character(),
sort %>% str_detect("naics") ~ glue("sortBy=PRINCIPAL_NAICS_CODE&{sort_dir}") %>% as.character(),
sort %>% str_detect("product code|service code") ~ glue("sortBy=PRODUCT_OR_SERVICE_CODE&{sort_dir}") %>% as.character(),
sort %>% str_detect("vendor state") ~ glue("sortBy=VENDOR_ADDRESS_STATE_CODE&{sort_dir}") %>% as.character(),
sort %>% str_detect("vendor zip") ~ glue("sortBy=VENDOR_ADDRESS_ZIP_CODE&{sort_dir}") %>% as.character(),
sort %>% str_detect("performance country") ~ glue("sortBy=POP_COUNTRY_NAME&{sort_dir}") %>% as.character(),
sort %>% str_detect("performance state") ~ glue("sortBy=POP_STATE_NAME&{sort_dir}") %>% as.character(),
sort %>% str_detect("local area") ~ glue("sortBy=LOCAL_AREA_SET_ASIDE&{sort_dir}") %>% as.character(),
sort %>% str_detect("treasury account") ~ glue("sortBy=TREASURY_ACCOUNT_SYMBOL&{sort_dir}") %>% as.character(),
sort %>% str_detect("local area") ~ glue("sortBy=LOCAL_AREA_SET_ASIDE&{sort_dir}") %>% as.character(),
sort %>% str_detect("non government") ~ glue("sortBy=NON_GOVERNMENT_VALUE&{sort_dir}") %>% as.character(),
sort %>% str_detect("contract fiscal year") ~ glue("sortBy=CONTRACT_FISCAL_YEAR&{sort_dir}") %>% as.character(),
TRUE ~ ""
)
sort_slug
}
.create_value_slug <-
function(values = "$100,000,000 | $200,000,000", escape = F) {
options(scipen = 999999)
values <- as.character(values)
values <-
values %>% str_split("\\|") %>% flatten_chr() %>% str_trim() %>% map_dbl(readr::parse_number) %>%
str_c(collapse = ",")
value_slug <- glue("[{values}]") %>% as.character()
if (escape) {
value_slug <- url_escape(value_slug)
}
value_slug
}
.shorten_dates <- function(dates = c("2018-01-01", "2019-03-01")) {
if (length(dates) == 1) {
dates <- c(dates, dates)
}
as.character(dates) %>% readr::parse_date() %>% as.character() %>% str_replace_all("\\-", "/") %>%
str_c(collapse = " | ")
}
.shorten_amounts <-
function(amounts = c(1000000, 20000000)) {
if (length(amounts) == 1) {
amounts <- c(amounts, amounts)
}
amounts %>% as.character() %>% map_dbl(readr::parse_number) %>% formattable::currency(digits = 0) %>%
as.character() %>%
str_c(collapse = " | ")
}
.shorten_count <-
function(amounts = c(10, 20)) {
if (length(amounts) == 1) {
amounts <- c(amounts, amounts)
}
amounts %>% as.character() %>% map_dbl(readr::parse_number) %>%
as.character() %>%
str_c(collapse = " | ")
}
.shorten_character <-
function(x = c("A", "B")) {
x %>%
str_c(collapse = " | ")
}
.count_results <-
function(url = "https://www.fpds.gov/ezsearch/fpdsportal?s=FPDS&q=%20AWARD_COMPLETION_DATE%3A%5B2019/04/02%2C2019/04/03%5D&indexName=awardfull&y=0&x=0&templateName=1.4") {
page <- url %>% read_html()
page %>% html_nodes("b~ b+ b") %>% html_text() %>% readr::parse_number()
}
.create_date_slug <-
function(dates = "2019/04/01 | 2019/04/03", escape = F) {
values <- as.character(dates)
values <-
values %>%
str_replace_all("\\-", "/") %>%
str_split("\\|") %>% flatten_chr() %>% str_trim() %>%
str_c(collapse = ",")
if (!dates %>% str_detect("|")) {
values <- str_c(values, values, sep = ",")
}
value_slug <- glue("[{values}]") %>% as.character()
if (escape) {
value_slug <- url_escape(value_slug)
}
value_slug
}
# list_munge --------------------------------------------------------------
.munge_double_list <-
function(result_list,
date_columns = c(
"VENDOR_RENEWAL_DATE",
"SOLICITATION_DATE",
"FINAL_INVOICE_PAID_DATE",
"SYS_LAST_MODIFIED_DATE",
"SIGNED_DATE",
"FUNDED_THROUGH_DATE",
"ESTIMATED_COMPLETION_DATE",
"LAST_MODIFIED_DATE",
"EFFECTIVE_DATE",
"CANCELLATION_DATE",
"LAST_DATE_TO_ORDER",
"PHYSICAL_COMPLETION_DATE",
"SOLICITATION_ISSUE_DATE",
"CREATED_DATE",
"DESTROY_DATE",
"AWARD_COMPLETION_DATE",
"LAST_MOD_DATE",
"VENDOR_REGISTRATION_DATE",
"REVEAL_DATE"
)
,
amount_columns = c(
"ANNUAL_REVENUE",
"BASE_EXERCISED_OPTIONS_VALUE",
"CURRENT_CONTRACT_VALUE",
"DOLLARS_OBLIGATED",
"CONTRACT_VALUE",
"FEE_RANGE_LOWER_VALUE",
"FEE_RANGE_UPPER_VALUE",
"FIXED_FEE_VALUE",
"NON_GOVERNMENT_VALUE",
"OBLIGATED_AMOUNT",
"TOTAL_CURRENT_CONTRACT_VALUE",
"TOTAL_DOLLARS_OBLIGATED",
"TOTAL_NON_GOVERNMENT_VALUE",
"TOTAL_ULTIMATE_CONTRACT_VALUE",
"ULTIMATE_CONTRACT_VALUE"
),
count_columns = c("NUMBER_OF_OFFERS_RECEIVED", "NUMBER_OF_EMPLOYEES")) {
parameters <- names(result_list)
values <- result_list %>% map_dbl(length) %>% as.numeric()
df_params <-
tibble(parameter = parameters, value = values)
if (df_params %>% filter(value > 1) %>% nrow() == 0) {
return(result_list)
}
double_values <-
df_params %>% filter(value > 1) %>% pull(parameter)
double_values <- double_values %>% str_to_upper()
double_values %>%
walk(function(double_value) {
double_value_slug <- str_to_lower(double_value)
value <- result_list[[double_value_slug]]
is_amount <-
double_value %in% amount_columns
is_count <-
double_value %in% count_columns
is_date <-
double_value %in% date_columns
if (is_amount) {
value <- value %>% .shorten_amounts()
result_list[[double_value_slug]] <<-
value
return(invisible())
}
if (is_date) {
value <-
value %>% .shorten_dates()
result_list[[double_value_slug]] <<-
value
return(invisible())
}
if (is_count) {
value <-
value %>% .shorten_count()
result_list[[double_value_slug]] <<-
value
return(invisible())
}
value <-
value %>% .shorten_character()
result_list[[double_value_slug]] <<-
value
return(invisible())
})
result_list
}
.create_fpds_query_params <-
function(result_list,
date_columns = c(
"VENDOR_RENEWAL_DATE",
"SOLICITATION_DATE",
"FINAL_INVOICE_PAID_DATE",
"SYS_LAST_MODIFIED_DATE",
"SIGNED_DATE",
"FUNDED_THROUGH_DATE",
"ESTIMATED_COMPLETION_DATE",
"LAST_MODIFIED_DATE",
"EFFECTIVE_DATE",
"CANCELLATION_DATE",
"LAST_DATE_TO_ORDER",
"PHYSICAL_COMPLETION_DATE",
"SOLICITATION_ISSUE_DATE",
"CREATED_DATE",
"DESTROY_DATE",
"AWARD_COMPLETION_DATE",
"LAST_MOD_DATE",
"VENDOR_REGISTRATION_DATE",
"REVEAL_DATE"
),
amount_columns = c(
"BASE_EXERCISED_OPTIONS_VALUE",
"CURRENT_CONTRACT_VALUE",
"DOLLARS_OBLIGATED",
"ANNUAL_REVENUE",
"CONTRACT_VALUE",
"FEE_RANGE_LOWER_VALUE",
"FEE_RANGE_UPPER_VALUE",
"FIXED_FEE_VALUE",
"NON_GOVERNMENT_VALUE",
"OBLIGATED_AMOUNT",
"TOTAL_CURRENT_CONTRACT_VALUE",
"TOTAL_DOLLARS_OBLIGATED",
"TOTAL_NON_GOVERNMENT_VALUE",
"TOTAL_ULTIMATE_CONTRACT_VALUE",
"ULTIMATE_CONTRACT_VALUE"
),
count_columns = c("NUMBER_OF_OFFERS_RECEIVED", "NUMBER_OF_EMPLOYEES")) {
df_params <-
result_list %>% flatten_df()
count_columns <-
count_columns %>% str_to_lower() %>% str_c(collapse = "|")
date_columns <-
date_columns %>% str_to_lower() %>% str_c(collapse = "|")
duns_columns <-
df_params %>% select(matches("duns")) %>% names()
amount_columns <-
amount_columns %>% str_to_lower() %>% str_c(collapse = "|")
date_names <-
df_params %>% select(matches(date_columns)) %>% names()
count_names <-
df_params %>% select(matches(count_columns)) %>% names()
amount_names <-
df_params %>% select(matches(amount_columns)) %>% names()
if (length(date_names) > 0) {
df_params <-
df_params %>%
mutate_at(date_names,
function(x) {
x %>% map_chr(.create_date_slug)
})
}
if (length(duns_columns) > 0) {
df_params <-
df_params %>%
mutate_at(date_names,
function(x) {
x %>% map_chr(.pad_fdps_duns)
})
}
if (length(amount_names) > 0) {
df_params <-
df_params %>%
mutate_at(amount_names,
function(x) {
x %>% map_chr(.create_value_slug)
})
}
if (length(count_names) > 0) {
df_params <-
df_params %>%
mutate_at(count_names,
function(x) {
x %>% map_chr(.create_value_slug)
})
}
df_params
}
# csv ---------------------------------------------------------------------
.read_fdps_csv <-
function(url = "https://www.fpds.gov/ezsearch/fpdsportal?s=FPDS.GOV&indexName=awardfull&templateName=CSV&q=%22Palantir+TECHNOLOGIES%22&renderer=jsp&length=677") {
options(warn = -1)
type_index <-
case_when(
url %>% str_to_lower() %>% str_detect("icd") ~ "ICD",
url %>% str_to_lower() %>% str_detect("initiative") ~ "RECOVERY",
TRUE ~ "AWARD"
)
data <- fread(url,
header = F,
verbose = F,
showProgress = F)
fpds_names <-
data %>% dplyr::slice(1) %>% t() %>% as.character()
actual_names <-
tibble(name = fpds_names) %>%
mutate(column = 1:n()) %>%
mutate(name = case_when(name %in% c("", NA) ~ str_c("X", column, sep = ""),
TRUE ~ name)) %>%
pull(name)
data <-
data %>%
dplyr::slice(2:nrow(data)) %>%
setNames(fpds_names) %>%
as_tibble(.name_repair = "unique")
select_cols <- 1:(ncol(data) - 1)
data <-
data %>%
select(select_cols)
if (type_index == "ICD") {
names(data)[[21]] <- c("nameNAICS")
names(data)[[23]] <- "nameFullPSC"
}
data <-
.munge_fpds_names(data = data)
if (data %>% hasName("amountObligation")) {
data <- data %>%
mutate(amountObligation = amountObligation %>% str_remove_all("\\$") %>% parse_number())
}
data <-
data %>%
mutate(urlCSV = url,
typeProcurement = type_index) %>%
dplyr::select(which(colMeans(is.na(.)) < 1)) %>%
dplyr::select(typeProcurement, everything())
if (data %>% hasName("codeProductService")) {
data <- data %>%
mutate(codeProductService = as.character(codeProductService))
}
data <-
data %>%
.fix_contract_name() %>%
.fix_contract_description()
data <-
data %>%
mutate_if(is.numeric, as.numeric) %>%
mutate_if(is.character,
list(function(x) {
case_when(x == "" ~ NA_character_,
TRUE ~ x)
}))
data
}
.fpds_single_csv_search <-
function(fpds_type = "award",
vendor_doing_business_as_name = NULL,
global_vendor_name = NULL,
vendor_name = NULL,
vendor_duns_number = NULL,
agency_name = NULL,
award_type = NULL,
solicitation_procedure = NULL,
contracting_office_name = NULL,
contracting_agency_name = NULL,
principal_naics_code = NULL,
award_status = NULL,
contract_type = NULL,
subcontract_plan = NULL,
contract_type_description = NULL,
type_of_contract_pricing = NULL,
contract_id = NULL,
ref_idv_contract_id = NULL,
ref_idv_agency_id = NULL,
modification_number = NULL,
contracting_agency_id = NULL,
contracting_office_id = NULL,
funding_agency_id = NULL,
funding_office_id = NULL,
funding_office_name = NULL,
agency_code = NULL,
department_id = NULL,
department_name = NULL,
last_mod_date = NULL,
last_modified_by = NULL,
award_completion_date = NULL,
created_date = NULL,
signed_date = NULL,
effective_date = NULL,
estimated_completion_date = NULL,
cancellation_date = NULL,
destroy_date = NULL,
final_invoice_paid_date = NULL,
funded_through_date = NULL,
last_modified_date = NULL,
physical_completion_date = NULL,
reveal_date = NULL,
solicitation_issue_date = NULL,
sys_last_modified_date = NULL,
vendor_registration_date = NULL,
vendor_renewal_date = NULL,
obligated_amount = NULL,
total_dollars_obligated = NULL,
current_contract_value = NULL,
total_current_contract_value = NULL,
base_exercised_options_value = NULL,
ultimate_contract_value = NULL,
total_ultimate_contract_value = NULL,
contract_fiscal_year = NULL,
created_by = NULL,
description_of_requirement = NULL,
reason_for_modification = NULL,
legislative_mandates = NULL,
local_area_set_aside = NULL,
socio_economic_indicators = NULL,
multiyear_contract = NULL,
national_interest_code = NULL,
national_interest_description = NULL,
naics_description = NULL,
product_or_service_code = NULL,
product_or_service_description = NULL,
performance_district_code = NULL,
performance_country = NULL,
performance_state_name = NULL,
parent_duns_number = NULL,
research = NULL,
vendor_address_city = NULL,
vendor_congress_district_code = NULL,
vendor_address_country_code = NULL,
vendor_address_country_name = NULL,
vendor_address_state_code = NULL,
vendor_address_state_name = NULL,
vendor_address_zip_code = NULL,
extent_competed = NULL,
number_of_offers_received = NULL,
dollars_obligated = NULL,
contract_value = NULL,
fee_range_lower_value = NULL,
fee_range_upper_value = NULL,
fixed_fee_value = NULL,
total_non_government_value = NULL,
sort_item = "Signed Date",
sort_descending = T,
return_message = T,
...) {
sort_slug <-
.sort_slug(sort_method = sort_item, sort_descending = sort_descending)
if (length(vendor_duns_number) > 0) {
vendor_duns_number <-
.pad_duns(duns = vendor_duns_number, zero_base = 9) %>% pull(slugDUNS)
}
if (length(parent_duns_number) > 0) {
parent_duns_number <-
.pad_duns(duns = parent_duns_number, zero_base = 9) %>% pull(slugDUNS)
}
query <-
.fpds_api_call_params(
contract_id = contract_id,
global_vendor_name = global_vendor_name,
contracting_agency_id = contracting_agency_id,
contracting_agency_name = contracting_agency_name,
contracting_office_id = contracting_office_id,
contract_type_description = contract_type_description,
contracting_office_name = contracting_office_name,
funding_agency_id = funding_agency_id,
funding_office_id = funding_office_id,
funding_office_name = funding_office_name,
agency_code = agency_code,
agency_name = agency_name,
department_id = department_id,
department_name = department_name,
last_mod_date = last_mod_date,
last_modified_by = last_modified_by,
award_completion_date = award_completion_date,
created_date = created_date,
signed_date = signed_date,
effective_date = effective_date,
estimated_completion_date = estimated_completion_date,
obligated_amount = obligated_amount,
ultimate_contract_value = ultimate_contract_value,
type_of_contract_pricing = type_of_contract_pricing,
award_status = award_status,
contract_type = contract_type,
description_of_requirement = description_of_requirement,
reason_for_modification = reason_for_modification,
legislative_mandates = legislative_mandates,
local_area_set_aside = local_area_set_aside,
socio_economic_indicators = socio_economic_indicators,
multiyear_contract = multiyear_contract,
national_interest_code = national_interest_code,
national_interest_description = national_interest_description,
principal_naics_code = principal_naics_code,
naics_description = naics_description,
product_or_service_code = product_or_service_code,
product_or_service_description = product_or_service_description,
performance_district_code = performance_district_code,
performance_country = performance_country,
pop_state_name = performance_state_name,
vendor_address_city = vendor_address_city,
vendor_congress_district_code = vendor_congress_district_code,
vendor_address_country_code = vendor_address_country_code,
vendor_address_country_name = vendor_address_country_name,
vendor_duns_number = vendor_duns_number,
vendor_doing_business_as_name = vendor_doing_business_as_name,
vendor_name = vendor_name,
vendor_address_state_code = vendor_address_state_code,
vendor_address_state_name = vendor_address_state_name,
vendor_address_zip_code = vendor_address_zip_code,
research = research,
cancellation_date = cancellation_date,
destroy_date = destroy_date,
final_invoice_paid_date = final_invoice_paid_date,
funded_through_date = funded_through_date,
last_modified_date = last_modified_date,
physical_completion_date = physical_completion_date,
reveal_date = reveal_date,
created_by = created_by,
solicitation_issue_date = solicitation_issue_date,
vendor_registration_date = vendor_registration_date,
base_exercised_options_value = base_exercised_options_value,
vendor_renewal_date = vendor_renewal_date,
sys_last_modified_date = vendor_renewal_date,
extent_competed = extent_competed,
return_query = T,
award_type = award_type,
solicitation_procedure = solicitation_procedure,
subcontract_plan = subcontract_plan,
number_of_offers_received = number_of_offers_received,
current_contract_value = current_contract_value ,
dollars_obligated = dollars_obligated,
contract_value = contract_value,
fee_range_lower_value = fee_range_lower_value,
fee_range_upper_value = fee_range_upper_value,
fixed_fee_value = fixed_fee_value,
total_current_contract_value = total_current_contract_value,
total_dollars_obligated = total_dollars_obligated,
total_non_government_value = total_non_government_value,
total_ultimate_contract_value = total_ultimate_contract_value,
contract_fiscal_year = contract_fiscal_year,
ref_idv_contract_id = ref_idv_contract_id,
parent_duns_number = parent_duns_number,
...
)
sort_slug <-
.sort_slug(sort_method = sort_item, sort_descending = sort_descending)
if (return_message) {
query %>% message()
}
fpds_type_slug <- fpds_type %>% str_to_lower()
if (!fpds_type_slug %in% c("award", "icd", "recovery")) {
stop("FPDS type can only be Award, ICD, or Recovery")
}
fpds_slug <-
case_when(
fpds_type_slug == "award" ~ "awardfull",
fpds_type_slug == "icd" ~ "icdindex",
TRUE ~ "initiative"
)
base_html <-
glue(
"https://www.fpds.gov/ezsearch/fpdsportal?indexName={fpds_slug}&templateName=1.5.1&s=FPDS.GOV"
) %>% as.character()
url_check <-
glue("{base_html}&q={query}") %>% as.character()
.count_results_safe <-
purrr::possibly(.count_results, 0)
results <- .count_results(url = url_check)
if (results == 0) {
"No Results" %>% message()
return(tibble())
}
max_results <- pmin(30000, results)
is_over_30k <- results > 30000
if (is_over_30k) {
glue(
"There are {comma(results, digits = 0)} results but you can only access 30,000 try detailing you search"
) %>%
message()
}
csv_slug <-
glue(
"https://www.fpds.gov/ezsearch/fpdsportal?s=fpds.gov&indexName={fpds_slug}&templateName=CSV"
)
if (is_over_30k) {
df_dates <- .generate_search_dates()
}
url <-
glue("{csv_slug}&q={query}&renderer=jsp&length={max_results}{sort_slug}") %>% as.character()
if (return_message) {
glue("Accessing {comma(results, digits = 0)} results for {URLdecode(query)}") %>% message()
}
data <-
.read_fdps_csv(url = url)
names(data) <- names(data) %>% str_replace_all("slugDUNS", "idDUNS")
duns <- names(data)[names(data) %>% str_detect("idDUNS")]
if (length(duns) > 0) {
data <- data %>%
mutate_at(duns, as.numeric)
}
data <- data %>%
munge_lite(unformat = T)
df_deps <-
data %>% distinct(nameAgencyAward)
df_deps <-
df_deps %>% separate(
nameAgencyAward,
sep = "\\,",
fill = "right",
into = c("part1", "part2"),
remove = F
) %>%
mutate_all(str_squish) %>%
mutate(nameAgencyAwardClean = case_when(is.na(part2) ~ part1,
!is.na(part2) ~ str_c(part2, part1, sep = " "))) %>%
select(nameAgencyAward, nameAgencyAwardClean)
data <-
data %>%
left_join(df_deps, by = "nameAgencyAward") %>%
select(-nameAgencyAward) %>%
rename(nameAgencyAward = nameAgencyAwardClean) %>%
select(one_of(names(data)), everything()) %>%
.add_budget_year()
date_cols <- data %>%
select(matches("date")) %>%
select_if(is.character) %>%
names()
if (length(date_cols) > 0) {
data <- data %>%
mutate_at(date_cols, mdy)
}
if (return_message) {
actions <- data %>% nrow() %>% comma(digits = 0)
contracts <-
data %>% distinct(idContractAnalysis) %>% nrow() %>% comma(digits = 0)
duns <-
data %>% distinct(idDUNS) %>% nrow() %>% comma(digits = 0)
parent_duns <-
data %>% distinct(idDUNSParent) %>% nrow() %>% comma(digits = 0)
amt <-
data$amountObligation %>% sum() %>% currency(digits = 0)
from_date <- data$dateContractSigned %>% min(na.rm = T)
to_date <- data$dateContractSigned %>% max(na.rm = T)
glue(
"\n\n{green({amt})} procured between {red({from_date})} and {red({to_date})} across {yellow(actions)} actions amongst {magenta(contracts)} contracts allocated to {cyan(duns)} distinct DUNS and {blue(parent_duns)} distinct parent DUNS\n\n"
) %>% message()
}
data %>%
mutate_if(is.numeric, as.numeric)
}
#' FPDS csv search
#'
#' Search FPDS returning CSV files.
#'
#'
#' @param fpds_type \itemize{
#' \item award
#' \item recovery
#' \item ICD
#' }
#' @param vendor_doing_business_as_name
#' @param global_vendor_name
#' @param vendor_name
#' @param parent_vendor_name
#' @param vendor_duns_number
#' @param parent_duns_number
#' @param agency_name
#' @param contracting_office_name
#' @param contracting_agency_name
#' @param principal_naics_code
#' @param award_status
#' @param contract_type
#' @param contract_type_description
#' @param type_of_contract_pricing
#' @param contract_id
#' @param ref_idv_contract_id
#' @param ref_idv_agency_id
#' @param contracting_agency_id
#' @param contracting_office_id
#' @param funding_agency_id
#' @param funding_office_id
#' @param funding_office_name
#' @param agency_code
#' @param department_id
#' @param department_name
#' @param research
#' @param last_mod_date
#' @param last_modified_by
#' @param award_completion_date
#' @param created_date
#' @param signed_date
#' @param effective_date
#' @param estimated_completion_date
#' @param obligated_amount
#' @param base_exercised_options_value
#' @param ultimate_contract_value
#' @param created_by
#' @param description_of_requirement
#' @param reason_for_modification
#' @param legislative_mandates
#' @param local_area_set_aside
#' @param socio_economic_indicators
#' @param multiyear_contract
#' @param national_interest_code
#' @param product_or_service_code
#' @param performance_district_code
#' @param performance_country
#' @param performance_state_name
#' @param vendor_address_city
#' @param vendor_congress_district_code
#' @param vendor_address_country_code
#' @param vendor_address_country_name
#' @param vendor_address_state_code
#' @param vendor_address_state_name
#' @param vendor_address_zip_code
#' @param sort_item
#' @param sort_descending
#' @param return_message
#' @param cancellation_date
#' @param destroy_date
#' @param final_invoice_paid_date
#' @param funded_through_date
#' @param last_modified_date
#' @param physical_completion_date
#' @param reveal_date
#' @param solicitation_issue_date
#' @param sys_last_modified_date
#' @param vendor_registration_date
#' @param vendor_renewal_date
#' @param award_type
#' @param solicitation_procedure
#' @param subcontract_plan
#' @param ...
#' @param current_contract_value
#' @param dollars_obligated
#' @param contract_value
#' @param fee_range_lower_value
#' @param fee_range_upper_value
#' @param fixed_fee_value
#' @param total_current_contract_value
#' @param total_dollars_obligated
#' @param total_non_government_value
#' @param total_ultimate_contract_value
#' @param contract_fiscal_year
#' @param extent_competed
#' @param number_of_offers_received
#' @param clean_entity_column if \code{TRUE} cleans entity columns
#' @param unformat
#' @param exclude_bloat
#' @param use_future
#' @param snake_names
#' @param decode_contract_ids
#'
#' @return
#' @export
#'
#' @examples
#' library(tidyverse)
#' library(govtrackR)
#'
#' ## Anything with contract signed between April 6th & 8th
#'
#' data <- fpds_csv(signed_date = "2019/04/06|2019/04/08")
#'
#'
#' ## What has the government spent on software from March 1st 2019 to April 8th 2019 vs Last Year
#'
#' df_software <- fpds_csv(principal_naics_code = 511210, signed_date = c("2019-03-01", "2019-04-08"))
#'
#' df_software %>% glimpse()
#'
#' ## Exhibit Design Architectural Services
#'
#' fpds_csv(product_or_service_code = "C1PB")
fpds_csv <-
function(fpds_type = "award",
decode_contract_ids = F,
vendor_doing_business_as_name = NA,
global_vendor_name = NA,
vendor_name = NA,
parent_vendor_name = NA,
base_exercised_options_value = NA,
current_contract_value = NA,
dollars_obligated = NA,
contract_value = NA,
fee_range_lower_value = NA,
fee_range_upper_value = NA,
fixed_fee_value = NA,
obligated_amount = NA,
total_current_contract_value = NA,
total_dollars_obligated = NA,
total_non_government_value = NA,
total_ultimate_contract_value = NA,
ultimate_contract_value = NA,
contract_fiscal_year = NA,
vendor_duns_number = NA,
parent_duns_number = NA,
agency_name = NA,
award_type = NA,
solicitation_procedure = NA,
contracting_office_name = NA,
contracting_agency_name = NA,
principal_naics_code = NA,
award_status = NA,
contract_type = NA,
contract_type_description = NA,
type_of_contract_pricing = NA,
contract_id = NA,
subcontract_plan = NA,
ref_idv_contract_id = NA,
ref_idv_agency_id = NA,
contracting_agency_id = NA,
contracting_office_id = NA,
funding_agency_id = NA,
funding_office_id = NA,
funding_office_name = NA,
agency_code = NA,
department_id = NA,
department_name = NA,
research = NA,
last_mod_date = NA,
last_modified_by = NA,
award_completion_date = NA,
created_date = NA,
signed_date = NA,
effective_date = NA,
estimated_completion_date = NA,
cancellation_date = NA,
destroy_date = NA,
final_invoice_paid_date = NA,
funded_through_date = NA,
last_modified_date = NA,
physical_completion_date = NA,
reveal_date = NA,
solicitation_issue_date = NA,
sys_last_modified_date = NA,
vendor_registration_date = NA,
vendor_renewal_date = NA,
created_by = NA,
description_of_requirement = NA,
reason_for_modification = NA,
legislative_mandates = NA,
local_area_set_aside = NA,
socio_economic_indicators = NA,
multiyear_contract = NA,
national_interest_code = NA,
product_or_service_code = NA,
performance_district_code = NA,
performance_country = NA,
performance_state_name = NA,
vendor_address_city = NA,
vendor_congress_district_code = NA,
vendor_address_country_code = NA,
vendor_address_country_name = NA,
vendor_address_state_code = NA,
vendor_address_state_name = NA,
vendor_address_zip_code = NA,
extent_competed = NA,
number_of_offers_received = NA,
sort_item = "Signed Date",
unformat = F,
exclude_bloat = T,
clean_entity_column = F,
sort_descending = T,
use_future = T,
return_message = T,
snake_names = F,
...) {
if (!is.na(contract_id) %>% sum(na.rm = T) > 0) {
contract_id <-
contract_id %>% str_remove_all("\\-")
}
if (!is.na(last_mod_date) %>% sum(na.rm = T) > 0 &
length(last_mod_date) == 2) {
last_mod_date <- .shorten_dates(dates = last_mod_date)
}
if (!is.na(award_completion_date) %>% sum(na.rm = T) > 0 &
length(award_completion_date) == 2) {
award_completion_date <-
.shorten_dates(dates = award_completion_date)
}
if (!is.na(created_date) %>% sum(na.rm = T) > 0 &
length(created_date) == 2) {
created_date <- .shorten_dates(dates = created_date)
}
if (!is.na(signed_date) %>% sum(na.rm = T) > 0 &
length(signed_date) == 2) {
signed_date <- .shorten_dates(dates = signed_date)
}
if (!is.na(effective_date) %>% sum(na.rm = T) > 0 &
length(effective_date) == 2) {
effective_date <- .shorten_dates(dates = effective_date)
}
if (!is.na(estimated_completion_date) %>% sum(na.rm = T) > 0 &
length(estimated_completion_date) == 2) {
estimated_completion_date <-
.shorten_dates(dates = estimated_completion_date)
}
if (!is.na(cancellation_date) %>% sum(na.rm = T) > 0 &
length(cancellation_date) == 2) {
cancellation_date <- .shorten_dates(dates = cancellation_date)
}
if (!is.na(destroy_date) %>% sum(na.rm = T) > 0 &
length(destroy_date) == 2) {
destroy_date <- .shorten_dates(dates = destroy_date)
}
if (!is.na(final_invoice_paid_date) %>% sum(na.rm = T) > 0 &
length(final_invoice_paid_date) == 2) {
final_invoice_paid_date <-
.shorten_dates(dates = final_invoice_paid_date)
}
if (!is.na(funded_through_date) %>% sum(na.rm = T) > 0 &
length(funded_through_date) == 2) {
funded_through_date <- .shorten_dates(dates = funded_through_date)
}
if (!is.na(last_modified_date) %>% sum(na.rm = T) > 0 &
length(last_modified_date) == 2) {
last_modified_date <- .shorten_dates(dates = last_modified_date)
}
if (!is.na(physical_completion_date) %>% sum(na.rm = T) > 0 &
length(physical_completion_date) == 2) {
physical_completion_date <-
.shorten_dates(dates = physical_completion_date)
}
if (!is.na(reveal_date) %>% sum(na.rm = T) > 0 &
length(reveal_date) == 2) {
reveal_date <- .shorten_dates(dates = reveal_date)
}
if (!is.na(solicitation_issue_date) %>% sum(na.rm = T) > 0 &
length(solicitation_issue_date) == 2) {
solicitation_issue_date <-
.shorten_dates(dates = solicitation_issue_date)
}
if (!is.na(sys_last_modified_date) %>% sum(na.rm = T) > 0 &
length(sys_last_modified_date) == 2) {
sys_last_modified_date <-
.shorten_dates(dates = sys_last_modified_date)
}
if (!is.na(vendor_registration_date) %>% sum(na.rm = T) > 0 &
length(vendor_registration_date) == 2) {
vendor_registration_date <-
.shorten_dates(dates = vendor_registration_date)
}
if (!is.na(vendor_renewal_date) %>% sum(na.rm = T) > 0 &
length(signed_date) == 2) {
vendor_renewal_date <-
.shorten_dates(dates = vendor_renewal_date)
}
if (!is.na(base_exercised_options_value) %>% sum(na.rm = T) > 0 &
length(base_exercised_options_value) == 2) {
base_exercised_options_value <-
.shorten_amounts(base_exercised_options_value)
}
if (!is.na(current_contract_value) %>% sum(na.rm = T) > 0 &
length(current_contract_value) == 2) {
current_contract_value <- .shorten_amounts(current_contract_value)
}
if (!is.na(dollars_obligated) %>% sum(na.rm = T) > 0 &
length(dollars_obligated) == 2) {
dollars_obligated <- .shorten_amounts(dollars_obligated)
}
if (!is.na(contract_value) %>% sum(na.rm = T) > 0 &
length(contract_value) == 2) {
contract_value <- .shorten_amounts(contract_value)
}
if (!is.na(fee_range_lower_value) %>% sum(na.rm = T) > 0 &
length(fee_range_lower_value) == 2) {
fee_range_lower_value <- .shorten_amounts(fee_range_lower_value)
}
if (!is.na(fee_range_upper_value) %>% sum(na.rm = T) > 0 &
length(fee_range_upper_value) == 2) {
fee_range_upper_value <- .shorten_amounts(fee_range_upper_value)
}
if (!is.na(fixed_fee_value) %>% sum(na.rm = T) > 0 &
length(fixed_fee_value) == 2) {
fixed_fee_value <- .shorten_amounts(fixed_fee_value)
}
if (!is.na(obligated_amount) %>% sum(na.rm = T) > 0 &
length(obligated_amount) == 2) {
obligated_amount <- .shorten_amounts(obligated_amount)
}
if (!is.na(total_current_contract_value) %>% sum(na.rm = T) > 0 &
length(total_current_contract_value) == 2) {
total_current_contract_value <-
.shorten_amounts(total_current_contract_value)
}
if (!is.na(total_dollars_obligated) %>% sum(na.rm = T) > 0 &
length(total_dollars_obligated) == 2) {
total_dollars_obligated <- .shorten_amounts(total_dollars_obligated)
}
if (!is.na(total_non_government_value) %>% sum(na.rm = T) > 0 &
length(total_non_government_value) == 2) {
total_non_government_value <-
.shorten_amounts(total_non_government_value)
}
if (!is.na(total_ultimate_contract_value) %>% sum(na.rm = T) > 0 &
length(total_ultimate_contract_value) == 2) {
total_ultimate_contract_value <-
.shorten_amounts(total_ultimate_contract_value)
}
if (!is.na(ultimate_contract_value) %>% sum(na.rm = T) > 0 &
length(ultimate_contract_value) == 2) {
ultimate_contract_value <-
.shorten_amounts(ultimate_contract_value)
}
df_inputs <-
expand.grid(
vendor_doing_business_as_name = vendor_doing_business_as_name,
global_vendor_name = global_vendor_name,
vendor_name = vendor_name,
award_type = award_type,
vendor_duns_number = vendor_duns_number,
agency_name = agency_name,
solicitation_procedure = solicitation_procedure,
contracting_office_name = contracting_office_name,
contracting_agency_name = contracting_agency_name,
principal_naics_code = principal_naics_code,
parent_duns_number = parent_duns_number,
parent_vendor_name = parent_vendor_name,
base_exercised_options_value = base_exercised_options_value,
current_contract_value = current_contract_value,
dollars_obligated = dollars_obligated,
contract_value = contract_value,
fee_range_lower_value = fee_range_lower_value,
fee_range_upper_value = fee_range_upper_value,
fixed_fee_value = fixed_fee_value,
obligated_amount = obligated_amount ,
total_current_contract_value = total_current_contract_value,
total_dollars_obligated = total_dollars_obligated,
total_non_government_value = total_non_government_value,
total_ultimate_contract_value = total_ultimate_contract_value,
ultimate_contract_value = ultimate_contract_value,
contract_fiscal_year = contract_fiscal_year,
award_status = award_status,
contract_type = contract_type,
contract_type_description = contract_type_description,
type_of_contract_pricing = type_of_contract_pricing,
contract_id = contract_id,
ref_idv_contract_id = ref_idv_contract_id,
ref_idv_agency_id = ref_idv_agency_id ,
contracting_agency_id = contracting_agency_id,
contracting_office_id = contracting_office_id,
funding_agency_id = funding_agency_id,
funding_office_id = funding_office_id,
funding_office_name = funding_office_name ,
agency_code = agency_code,
department_id = department_id,
department_name = department_name,
last_mod_date = last_mod_date,
last_modified_by = last_modified_by,
award_completion_date = award_completion_date,
created_date = created_date,
signed_date = signed_date ,
effective_date = effective_date,
estimated_completion_date = estimated_completion_date,
cancellation_date = cancellation_date,
destroy_date = destroy_date,
final_invoice_paid_date = final_invoice_paid_date,
funded_through_date = funded_through_date,
last_modified_date = last_modified_date,
physical_completion_date = physical_completion_date,
reveal_date = reveal_date,
solicitation_issue_date = solicitation_issue_date,
sys_last_modified_date = sys_last_modified_date,
vendor_registration_date = vendor_registration_date,
vendor_renewal_date = vendor_renewal_date,
created_by = created_by,
description_of_requirement = description_of_requirement,
reason_for_modification = reason_for_modification ,
legislative_mandates = legislative_mandates,
local_area_set_aside = local_area_set_aside ,
socio_economic_indicators = socio_economic_indicators ,
multiyear_contract = multiyear_contract,
national_interest_code = national_interest_code,
product_or_service_code = product_or_service_code,
performance_district_code = performance_district_code,
performance_country = performance_country ,
performance_state_name = performance_state_name,
vendor_address_city = vendor_address_city,
vendor_congress_district_code = vendor_congress_district_code,
vendor_address_country_code = vendor_address_country_code,
vendor_address_country_name = vendor_address_country_name,
vendor_address_state_code = vendor_address_state_code,
vendor_address_state_name = vendor_address_state_name,
vendor_address_zip_code = vendor_address_zip_code,
research = research,
subcontract_plan = subcontract_plan,
extent_competed = extent_competed,
number_of_offers_received = number_of_offers_received,
stringsAsFactors = F,
...
) %>%
as_tibble()
if (df_inputs %>% gather(item, value) %>% filter(!is.na(value)) %>% nrow() == 0) {
stop("Please enter FPDS search parameter")
}
.fpds_single_csv_search_safe <-
possibly(.fpds_single_csv_search, tibble())
if (use_future) {
all_data <-
all_data <-
1:nrow(df_inputs) %>%
future_map_dfr(function(x) {
df_row <-
df_inputs %>% dplyr::slice(x) %>% as_tibble() %>% mutate_if(is.factor, as.character)
if (is.na(df_row$vendor_doing_business_as_name)) {
vendor_doing_business_as_name_slug <- NULL
} else {
vendor_doing_business_as_name_slug <-
df_row$vendor_doing_business_as_name
}
if (is.na(df_row$global_vendor_name)) {
global_vendor_name_slug <- NULL
} else {
global_vendor_name_slug <- df_row$global_vendor_name
}
if (is.na(df_row$solicitation_procedure)) {
solicitation_procedure_slug <- NULL
} else {
solicitation_procedure_slug <- df_row$solicitation_procedure
}
if (is.na(df_row$subcontract_plan)) {
subcontract_plan_slug <- NULL
} else {
subcontract_plan_slug <- df_row$subcontract_plan
}
if (is.na(df_row$award_type)) {
award_type_slug <- NULL
} else {
award_type_slug <- df_row$award_type
}
if (is.na(df_row$research)) {
research_slug <- NULL
} else {
research_slug <- df_row$research
}
if (is.na(df_row$vendor_name)) {
vendor_name_slug <- NULL
} else {
vendor_name_slug <- df_row$vendor_name
}
if (is.na(df_row$parent_vendor_name)) {
parent_vendor_name_slug <- NULL
} else {
parent_vendor_name_slug <- df_row$parent_vendor_name
}
if (is.na(df_row$vendor_duns_number)) {
vendor_duns_number_slug <- NULL
} else {
vendor_duns_number_slug <- df_row$vendor_duns_number
}
if (is.na(df_row$parent_duns_number)) {
parent_duns_number_slug <- NULL
} else {
parent_duns_number_slug <- df_row$parent_duns_number
}
if (is.na(df_row$agency_name)) {
agency_name_slug <- NULL
} else {
agency_name_slug <- df_row$agency_name
}
if (is.na(df_row$contracting_office_name)) {
contracting_office_name_slug <- NULL
} else {
contracting_office_name_slug <- df_row$contracting_office_name
}
if (is.na(df_row$contracting_agency_name)) {
contracting_agency_name_slug <- NULL
} else {
contracting_agency_name_slug <- df_row$contracting_agency_name
}
if (is.na(df_row$principal_naics_code)) {
principal_naics_code_slug <- NULL
} else {
principal_naics_code_slug <- df_row$principal_naics_code
}
if (is.na(df_row$award_status)) {
award_status_slug <- NULL
} else {
award_status_slug <- df_row$award_status
}
if (is.na(df_row$contract_type)) {
contract_type_slug <- NULL
} else {
contract_type_slug <- df_row$contract_type
}
if (is.na(df_row$contract_type_description)) {
contract_type_description_slug <- NULL
} else {
contract_type_description_slug <- df_row$contract_type_description
}
if (is.na(df_row$type_of_contract_pricing)) {
type_of_contract_pricing_slug <- NULL
} else {
type_of_contract_pricing_slug <- df_row$type_of_contract_pricing
}
if (is.na(df_row$contract_id)) {
contract_id_slug <- NULL
} else {
contract_id_slug <- df_row$contract_id
}
if (is.na(df_row$ref_idv_contract_id)) {
ref_idv_contract_id_slug <- NULL
} else {
ref_idv_contract_id_slug <- df_row$ref_idv_contract_id
}
if (is.na(df_row$ref_idv_agency_id)) {
ref_idv_agency_id_slug <- NULL
} else {
ref_idv_agency_id_slug <- df_row$ref_idv_agency_id
}
if (is.na(df_row$contracting_agency_id)) {
contracting_agency_id_slug <- NULL
} else {
contracting_agency_id_slug <- df_row$contracting_agency_id
}
if (is.na(df_row$contracting_office_id)) {
contracting_office_id_slug <- NULL
} else {
contracting_office_id_slug <- df_row$contracting_office_id
}
if (is.na(df_row$funding_agency_id)) {
funding_agency_id_slug <- NULL
} else {
funding_agency_id_slug <- df_row$funding_agency_id
}
if (is.na(df_row$funding_office_id)) {
funding_office_id_slug <- NULL
} else {
funding_office_id_slug <- df_row$funding_office_id
}
if (is.na(df_row$funding_office_name)) {
funding_office_name_slug <- NULL
} else {
funding_office_name_slug <- df_row$funding_office_name
}
if (is.na(df_row$agency_code)) {
agency_code_slug <- NULL
} else {
agency_code_slug <- df_row$agency_code
}
if (is.na(df_row$department_id)) {
department_id_slug <- NULL
} else {
department_id_slug <- df_row$department_id
}
if (is.na(df_row$department_name)) {
department_name_slug <- NULL
} else {
department_name_slug <- df_row$department_name
}
if (is.na(df_row$last_mod_date)) {
last_mod_date_slug <- NULL
} else {
last_mod_date_slug <- df_row$last_mod_date
}
if (is.na(df_row$last_modified_by)) {
last_modified_by_slug <- NULL
} else {
last_modified_by_slug <- df_row$last_modified_by
}
if (is.na(df_row$award_completion_date)) {
award_completion_date_slug <- NULL
} else {
award_completion_date_slug <- df_row$award_completion_date
}
if (is.na(df_row$created_date)) {
created_date_slug <- NULL
} else {
created_date_slug <- df_row$created_date
}
if (is.na(df_row$signed_date)) {
signed_date_slug <- NULL
} else {
signed_date_slug <- df_row$signed_date
}
if (is.na(df_row$effective_date)) {
effective_date_slug <- NULL
} else {
effective_date_slug <- df_row$effective_date
}
if (is.na(df_row$estimated_completion_date)) {
estimated_completion_date_slug <- NULL
} else {
estimated_completion_date_slug <- df_row$estimated_completion_date
}
if (is.na(df_row$cancellation_date)) {
cancellation_date_slug <- NULL
} else {
cancellation_date_slug <- df_row$cancellation_date
}
if (is.na(df_row$destroy_date)) {
destroy_date_slug <- NULL
} else {
destroy_date_slug <- df_row$destroy_date
}
if (is.na(df_row$final_invoice_paid_date)) {
final_invoice_paid_date_slug <- NULL
} else {
final_invoice_paid_date_slug <- df_row$final_invoice_paid_date
}
if (is.na(df_row$funded_through_date)) {
funded_through_date_slug <- NULL
} else {
funded_through_date_slug <- df_row$funded_through_date
}
if (is.na(df_row$last_modified_date)) {
last_modified_date_slug <- NULL
} else {
last_modified_date_slug <- df_row$last_modified_date
}
if (is.na(df_row$physical_completion_date)) {
physical_completion_date_slug <- NULL
} else {
physical_completion_date_slug <- df_row$physical_completion_date
}
if (is.na(df_row$reveal_date)) {
reveal_date_slug <- NULL
} else {
reveal_date_slug <- df_row$reveal_date
}
if (is.na(df_row$solicitation_issue_date)) {
solicitation_issue_date_slug <- NULL
} else {
solicitation_issue_date_slug <- df_row$solicitation_issue_date
}
if (is.na(df_row$sys_last_modified_date)) {
sys_last_modified_date_slug <- NULL
} else {
sys_last_modified_date_slug <- df_row$sys_last_modified_date
}
if (is.na(df_row$vendor_registration_date)) {
vendor_registration_date_slug <- NULL
} else {
vendor_registration_date_slug <- df_row$vendor_registration_date
}
if (is.na(df_row$vendor_renewal_date)) {
vendor_renewal_date_slug <- NULL
} else {
vendor_renewal_date_slug <-
df_row$vendor_revendor_renewal_dategistration_date
}
if (is.na(df_row$obligated_amount)) {
obligated_amount_slug <- NULL
} else {
obligated_amount_slug <- df_row$obligated_amount
}
if (is.na(df_row$base_exercised_options_value)) {
base_exercised_options_value_slug <- NULL
} else {
base_exercised_options_value_slug <-
df_row$base_exercised_options_value
}
if (is.na(df_row$ultimate_contract_value)) {
ultimate_contract_value_slug <- NULL
} else {
ultimate_contract_value_slug <- df_row$ultimate_contract_value
}
if (is.na(df_row$created_by)) {
created_by_slug <- NULL
} else {
created_by_slug <- df_row$created_by
}
if (is.na(df_row$description_of_requirement)) {
description_of_requirement_slug <- NULL
} else {
description_of_requirement_slug <- df_row$description_of_requirement
}
if (is.na(df_row$legislative_mandates)) {
legislative_mandates_slug <- NULL
} else {
legislative_mandates_slug <- df_row$legislative_mandates
}
if (is.na(df_row$local_area_set_aside)) {
local_area_set_aside_slug <- NULL
} else {
local_area_set_aside_slug <- df_row$local_area_set_aside
}
if (is.na(df_row$socio_economic_indicators)) {
socio_economic_indicators_slug <- NULL
} else {
socio_economic_indicators_slug <- df_row$socio_economic_indicators
}
if (is.na(df_row$multiyear_contract)) {
multiyear_contract_slug <- NULL
} else {
multiyear_contract_slug <- df_row$multiyear_contract
}
if (is.na(df_row$national_interest_code)) {
national_interest_code_slug <- NULL
} else {
national_interest_code_slug <- df_row$national_interest_code
}
if (is.na(df_row$product_or_service_code)) {
product_or_service_code_slug <- NULL
} else {
product_or_service_code_slug <- df_row$product_or_service_code
}
if (is.na(df_row$performance_district_code)) {
performance_district_code_slug <- NULL
} else {
performance_district_code_slug <- df_row$performance_district_code
}
if (is.na(df_row$performance_country)) {
performance_country_slug <- NULL
} else {
performance_country_slug <- df_row$performance_country
}
if (is.na(df_row$performance_state_name)) {
performance_state_name_slug <- NULL
} else {
performance_state_name_slug <- df_row$performance_state_name
}
if (is.na(df_row$vendor_address_city)) {
vendor_address_city_slug <- NULL
} else {
vendor_address_city_slug <- df_row$vendor_address_city
}
if (is.na(df_row$vendor_congress_district_code)) {
vendor_congress_district_code_slug <- NULL
} else {
vendor_congress_district_code_slug <-
df_row$vendor_congress_district_code
}
if (is.na(df_row$vendor_address_country_code)) {
vendor_address_country_code_slug <- NULL
} else {
vendor_address_country_code_slug <-
df_row$vendor_address_country_code
}
if (is.na(vendor_address_country_name)) {
vendor_address_country_name_slug <- NULL
} else {
vendor_address_country_name_slug <-
df_row$vendor_address_country_name
}
if (is.na(df_row$vendor_address_state_code)) {
vendor_address_state_code_slug <- NULL
} else {
vendor_address_state_code_slug <- df_row$vendor_address_state_code
}
if (is.na(df_row$vendor_address_state_name)) {
vendor_address_state_name_slug <- NULL
} else {
vendor_address_state_name_slug <- df_row$vendor_address_state_name
}
if (is.na(df_row$vendor_address_zip_code)) {
vendor_address_zip_code_slug <- NULL
} else {
vendor_address_zip_code_slug <- df_row$vendor_address_zip_code
}
if (is.na(df_row$reason_for_modification)) {
reason_for_modification_slug <- NULL
} else {
reason_for_modification_slug <- df_row$reason_for_modification
}
if (is.na(current_contract_value)) {
current_contract_value_slug <- NULL
} else {
current_contract_value_slug <- df_row$current_contract_value
}
if (is.na(dollars_obligated)) {
dollars_obligated_slug <- NULL
} else {
dollars_obligated_slug <- df_row$dollars_obligated
}
if (is.na(contract_value)) {
contract_value_slug <- NULL
} else {
contract_value_slug <- df_row$contract_value
}
if (is.na(fee_range_lower_value)) {
fee_range_lower_value_slug <- NULL
} else {
fee_range_lower_value_slug <- df_row$fee_range_lower_value
}
if (is.na(fee_range_upper_value)) {
fee_range_upper_value_slug <- NULL
} else {
}
if (is.na(fixed_fee_value)) {
fixed_fee_value_slug <- NULL
} else {
fixed_fee_value_slug <- df_row$fixed_fee_value
}
if (is.na(total_current_contract_value)) {
total_current_contract_value_slug <- NULL
} else {
total_current_contract_value_slug <-
df_row$total_current_contract_value
}
if (is.na(total_dollars_obligated)) {
total_dollars_obligated_slug <- NULL
} else {
total_dollars_obligated_slug <- df_row$total_dollars_obligated
}
if (is.na(total_non_government_value)) {
total_non_government_value_slug <- NULL
} else {
total_non_government_value_slug <- df_row$total_non_government_value
}
if (is.na(total_ultimate_contract_value)) {
total_ultimate_contract_value_slug <- NULL
} else {
total_ultimate_contract_value_slug <-
df_row$total_ultimate_contract_value
}
if (is.na(contract_fiscal_year)) {
contract_fiscal_year_slug <- NULL
} else {
contract_fiscal_year_slug <- df_row$contract_fiscal_year
}
if (is.na(extent_competed)) {
extent_competed_slug <- NULL
} else {
extent_competed_slug <- df_row$extent_competed
}
if (is.na(number_of_offers_received)) {
number_of_offers_received_slug <- NULL
} else {
number_of_offers_received_slug <- df_row$number_of_offers_received
}
d <-
.fpds_single_csv_search_safe(
fpds_type = fpds_type,
vendor_doing_business_as_name = vendor_doing_business_as_name_slug,
award_type = award_type_slug,
global_vendor_name = global_vendor_name_slug,
vendor_name = vendor_name_slug,
parent_vendor_name = parent_vendor_name_slug,
vendor_duns_number = vendor_duns_number_slug,
parent_duns_number = parent_duns_number_slug,
agency_name = agency_name_slug,
contracting_office_name = contracting_office_name_slug,
contracting_agency_name = contracting_agency_name_slug,
principal_naics_code = principal_naics_code_slug,
award_status = award_status_slug,
contract_type = contract_type_slug,
contract_type_description = contract_type_description_slug,
type_of_contract_pricing = type_of_contract_pricing_slug,
contract_id = contract_id_slug,
ref_idv_contract_id = ref_idv_contract_id_slug,
ref_idv_agency_id = ref_idv_agency_id_slug,
contracting_agency_id = contracting_agency_id_slug,
contracting_office_id = contracting_office_id_slug,
funding_agency_id = funding_agency_id_slug,
funding_office_id = funding_office_id_slug,
funding_office_name = funding_office_name_slug,
agency_code = agency_code_slug,
department_id = department_id_slug,
department_name = department_name_slug,
research = research_slug,
last_mod_date = last_mod_date_slug,
last_modified_by = last_modified_by_slug,
award_completion_date = award_completion_date_slug,
created_date = created_date_slug,
signed_date = signed_date_slug,
effective_date = effective_date_slug,
estimated_completion_date = estimated_completion_date_slug,
obligated_amount = obligated_amount_slug,
base_exercised_options_value = base_exercised_options_value_slug,
ultimate_contract_value = ultimate_contract_value_slug,
created_by = created_by_slug,
description_of_requirement = description_of_requirement_slug,
reason_for_modification = reason_for_modification_slug,
legislative_mandates = legislative_mandates_slug,
local_area_set_aside = local_area_set_aside_slug,
socio_economic_indicators = socio_economic_indicators_slug,
multiyear_contract = multiyear_contract_slug,
national_interest_code = national_interest_code_slug,
product_or_service_code = product_or_service_code_slug,
performance_district_code = performance_district_code_slug,
performance_country = performance_country_slug,
performance_state_name = performance_state_name_slug,
vendor_address_city = vendor_address_city_slug,
vendor_congress_district_code = vendor_congress_district_code_slug,
vendor_address_country_code = vendor_address_country_code_slug,
vendor_address_country_name = vendor_address_country_name_slug,
vendor_address_state_code = vendor_address_state_code_slug,
vendor_address_state_name = vendor_address_state_name_slug,
vendor_address_zip_code = vendor_address_zip_code_slug,
sort_item = sort_item,
sort_descending = sort_descending,
return_message = return_message,
cancellation_date = cancellation_date_slug,
destroy_date = destroy_date_slug,
final_invoice_paid_date = final_invoice_paid_date_slug,
funded_through_date = funded_through_date_slug,
last_modified_date = last_modified_date_slug,
physical_completion_date = physical_completion_date_slug,
reveal_date = reveal_date_slug,
solicitation_issue_date = solicitation_issue_date_slug,
sys_last_modified_date = sys_last_modified_date_slug,
vendor_registration_date = vendor_registration_date_slug,
vendor_renewal_date = vendor_renewal_date_slug,
solicitation_procedure = solicitation_procedure_slug,
subcontract_plan = subcontract_plan_slug,
total_dollars_obligated = total_dollars_obligated_slug,
current_contract_value = current_contract_value_slug,
total_current_contract_value = total_current_contract_value_slug,
total_ultimate_contract_value = total_ultimate_contract_value_slug,
contract_fiscal_year = contract_fiscal_year_slug,
...
)
if (d %>% hasName("codeModification")) {
d <- d %>%
mutate(codeModification = as.character(codeModification))
}
if (d %>% tibble::has_name("idAgencyAward")) {
d <- d %>%
mutate(idAgencyAward = as.character(idAgencyAward))
}
if (!d %>% tibble::has_name("zipcodeVendor")) {
d <- d %>%
mutate(zipcodeVendor = NA_character_)
}
if (d %>% hasName("codeProductService")) {
d <- d %>%
mutate(codeProductService = as.character(codeProductService))
}
d <- d %>%
mutate(zipcodeVendor = as.character(zipcodeVendor))
d
})
} else {
all_data <-
1:nrow(df_inputs) %>%
map_dfr(function(x) {
df_row <-
df_inputs %>% dplyr::slice(x) %>% as_tibble() %>% mutate_if(is.factor, as.character)
if (is.na(df_row$vendor_doing_business_as_name)) {
vendor_doing_business_as_name_slug <- NULL
} else {
vendor_doing_business_as_name_slug <-
df_row$vendor_doing_business_as_name
}
if (is.na(df_row$global_vendor_name)) {
global_vendor_name_slug <- NULL
} else {
global_vendor_name_slug <- df_row$global_vendor_name
}
if (is.na(df_row$solicitation_procedure)) {
solicitation_procedure_slug <- NULL
} else {
solicitation_procedure_slug <- df_row$solicitation_procedure
}
if (is.na(df_row$subcontract_plan)) {
subcontract_plan_slug <- NULL
} else {
subcontract_plan_slug <- df_row$subcontract_plan
}
if (is.na(df_row$award_type)) {
award_type_slug <- NULL
} else {
award_type_slug <- df_row$award_type
}
if (is.na(df_row$research)) {
research_slug <- NULL
} else {
research_slug <- df_row$research
}
if (is.na(df_row$vendor_name)) {
vendor_name_slug <- NULL
} else {
vendor_name_slug <- df_row$vendor_name
}
if (is.na(df_row$parent_vendor_name)) {
parent_vendor_name_slug <- NULL
} else {
parent_vendor_name_slug <- df_row$parent_vendor_name
}
if (is.na(df_row$vendor_duns_number)) {
vendor_duns_number_slug <- NULL
} else {
vendor_duns_number_slug <- df_row$vendor_duns_number
}
if (is.na(df_row$parent_duns_number)) {
parent_duns_number_slug <- NULL
} else {
parent_duns_number_slug <- df_row$parent_duns_number
}
if (is.na(df_row$agency_name)) {
agency_name_slug <- NULL
} else {
agency_name_slug <- df_row$agency_name
}
if (is.na(df_row$contracting_office_name)) {
contracting_office_name_slug <- NULL
} else {
contracting_office_name_slug <- df_row$contracting_office_name
}
if (is.na(df_row$contracting_agency_name)) {
contracting_agency_name_slug <- NULL
} else {
contracting_agency_name_slug <- df_row$contracting_agency_name
}
if (is.na(df_row$principal_naics_code)) {
principal_naics_code_slug <- NULL
} else {
principal_naics_code_slug <- df_row$principal_naics_code
}
if (is.na(df_row$award_status)) {
award_status_slug <- NULL
} else {
award_status_slug <- df_row$award_status
}
if (is.na(df_row$contract_type)) {
contract_type_slug <- NULL
} else {
contract_type_slug <- df_row$contract_type
}
if (is.na(df_row$contract_type_description)) {
contract_type_description_slug <- NULL
} else {
contract_type_description_slug <- df_row$contract_type_description
}
if (is.na(df_row$type_of_contract_pricing)) {
type_of_contract_pricing_slug <- NULL
} else {
type_of_contract_pricing_slug <- df_row$type_of_contract_pricing
}
if (is.na(df_row$contract_id)) {
contract_id_slug <- NULL
} else {
contract_id_slug <- df_row$contract_id
}
if (is.na(df_row$ref_idv_contract_id)) {
ref_idv_contract_id_slug <- NULL
} else {
ref_idv_contract_id_slug <- df_row$ref_idv_contract_id
}
if (is.na(df_row$ref_idv_agency_id)) {
ref_idv_agency_id_slug <- NULL
} else {
ref_idv_agency_id_slug <- df_row$ref_idv_agency_id
}
if (is.na(df_row$contracting_agency_id)) {
contracting_agency_id_slug <- NULL
} else {
contracting_agency_id_slug <- df_row$contracting_agency_id
}
if (is.na(df_row$contracting_office_id)) {
contracting_office_id_slug <- NULL
} else {
contracting_office_id_slug <- df_row$contracting_office_id
}
if (is.na(df_row$funding_agency_id)) {
funding_agency_id_slug <- NULL
} else {
funding_agency_id_slug <- df_row$funding_agency_id
}
if (is.na(df_row$funding_office_id)) {
funding_office_id_slug <- NULL
} else {
funding_office_id_slug <- df_row$funding_office_id
}
if (is.na(df_row$funding_office_name)) {
funding_office_name_slug <- NULL
} else {
funding_office_name_slug <- df_row$funding_office_name
}
if (is.na(df_row$agency_code)) {
agency_code_slug <- NULL
} else {
agency_code_slug <- df_row$agency_code
}
if (is.na(df_row$department_id)) {
department_id_slug <- NULL
} else {
department_id_slug <- df_row$department_id
}
if (is.na(df_row$department_name)) {
department_name_slug <- NULL
} else {
department_name_slug <- df_row$department_name
}
if (is.na(df_row$last_mod_date)) {
last_mod_date_slug <- NULL
} else {
last_mod_date_slug <- df_row$last_mod_date
}
if (is.na(df_row$last_modified_by)) {
last_modified_by_slug <- NULL
} else {
last_modified_by_slug <- df_row$last_modified_by
}
if (is.na(df_row$award_completion_date)) {
award_completion_date_slug <- NULL
} else {
award_completion_date_slug <- df_row$award_completion_date
}
if (is.na(df_row$created_date)) {
created_date_slug <- NULL
} else {
created_date_slug <- df_row$created_date
}
if (is.na(df_row$signed_date)) {
signed_date_slug <- NULL
} else {
signed_date_slug <- df_row$signed_date
}
if (is.na(df_row$effective_date)) {
effective_date_slug <- NULL
} else {
effective_date_slug <- df_row$effective_date
}
if (is.na(df_row$estimated_completion_date)) {
estimated_completion_date_slug <- NULL
} else {
estimated_completion_date_slug <- df_row$estimated_completion_date
}
if (is.na(df_row$cancellation_date)) {
cancellation_date_slug <- NULL
} else {
cancellation_date_slug <- df_row$cancellation_date
}
if (is.na(df_row$destroy_date)) {
destroy_date_slug <- NULL
} else {
destroy_date_slug <- df_row$destroy_date
}
if (is.na(df_row$final_invoice_paid_date)) {
final_invoice_paid_date_slug <- NULL
} else {
final_invoice_paid_date_slug <- df_row$final_invoice_paid_date
}
if (is.na(df_row$funded_through_date)) {
funded_through_date_slug <- NULL
} else {
funded_through_date_slug <- df_row$funded_through_date
}
if (is.na(df_row$last_modified_date)) {
last_modified_date_slug <- NULL
} else {
last_modified_date_slug <- df_row$last_modified_date
}
if (is.na(df_row$physical_completion_date)) {
physical_completion_date_slug <- NULL
} else {
physical_completion_date_slug <- df_row$physical_completion_date
}
if (is.na(df_row$reveal_date)) {
reveal_date_slug <- NULL
} else {
reveal_date_slug <- df_row$reveal_date
}
if (is.na(df_row$solicitation_issue_date)) {
solicitation_issue_date_slug <- NULL
} else {
solicitation_issue_date_slug <- df_row$solicitation_issue_date
}
if (is.na(df_row$sys_last_modified_date)) {
sys_last_modified_date_slug <- NULL
} else {
sys_last_modified_date_slug <- df_row$sys_last_modified_date
}
if (is.na(df_row$vendor_registration_date)) {
vendor_registration_date_slug <- NULL
} else {
vendor_registration_date_slug <- df_row$vendor_registration_date
}
if (is.na(df_row$vendor_renewal_date)) {
vendor_renewal_date_slug <- NULL
} else {
vendor_renewal_date_slug <-
df_row$vendor_revendor_renewal_dategistration_date
}
if (is.na(df_row$obligated_amount)) {
obligated_amount_slug <- NULL
} else {
obligated_amount_slug <- df_row$obligated_amount
}
if (is.na(df_row$base_exercised_options_value)) {
base_exercised_options_value_slug <- NULL
} else {
base_exercised_options_value_slug <-
df_row$base_exercised_options_value
}
if (is.na(df_row$ultimate_contract_value)) {
ultimate_contract_value_slug <- NULL
} else {
ultimate_contract_value_slug <- df_row$ultimate_contract_value
}
if (is.na(df_row$created_by)) {
created_by_slug <- NULL
} else {
created_by_slug <- df_row$created_by
}
if (is.na(df_row$description_of_requirement)) {
description_of_requirement_slug <- NULL
} else {
description_of_requirement_slug <- df_row$description_of_requirement
}
if (is.na(df_row$legislative_mandates)) {
legislative_mandates_slug <- NULL
} else {
legislative_mandates_slug <- df_row$legislative_mandates
}
if (is.na(df_row$local_area_set_aside)) {
local_area_set_aside_slug <- NULL
} else {
local_area_set_aside_slug <- df_row$local_area_set_aside
}
if (is.na(df_row$socio_economic_indicators)) {
socio_economic_indicators_slug <- NULL
} else {
socio_economic_indicators_slug <- df_row$socio_economic_indicators
}
if (is.na(df_row$multiyear_contract)) {
multiyear_contract_slug <- NULL
} else {
multiyear_contract_slug <- df_row$multiyear_contract
}
if (is.na(df_row$national_interest_code)) {
national_interest_code_slug <- NULL
} else {
national_interest_code_slug <- df_row$national_interest_code
}
if (is.na(df_row$product_or_service_code)) {
product_or_service_code_slug <- NULL
} else {
product_or_service_code_slug <- df_row$product_or_service_code
}
if (is.na(df_row$performance_district_code)) {
performance_district_code_slug <- NULL
} else {
performance_district_code_slug <- df_row$performance_district_code
}
if (is.na(df_row$performance_country)) {
performance_country_slug <- NULL
} else {
performance_country_slug <- df_row$performance_country
}
if (is.na(df_row$performance_state_name)) {
performance_state_name_slug <- NULL
} else {
performance_state_name_slug <- df_row$performance_state_name
}
if (is.na(df_row$vendor_address_city)) {
vendor_address_city_slug <- NULL
} else {
vendor_address_city_slug <- df_row$vendor_address_city
}
if (is.na(df_row$vendor_congress_district_code)) {
vendor_congress_district_code_slug <- NULL
} else {
vendor_congress_district_code_slug <-
df_row$vendor_congress_district_code
}
if (is.na(df_row$vendor_address_country_code)) {
vendor_address_country_code_slug <- NULL
} else {
vendor_address_country_code_slug <-
df_row$vendor_address_country_code
}
if (is.na(vendor_address_country_name)) {
vendor_address_country_name_slug <- NULL
} else {
vendor_address_country_name_slug <-
df_row$vendor_address_country_name
}
if (is.na(df_row$vendor_address_state_code)) {
vendor_address_state_code_slug <- NULL
} else {
vendor_address_state_code_slug <- df_row$vendor_address_state_code
}
if (is.na(df_row$vendor_address_state_name)) {
vendor_address_state_name_slug <- NULL
} else {
vendor_address_state_name_slug <- df_row$vendor_address_state_name
}
if (is.na(df_row$vendor_address_zip_code)) {
vendor_address_zip_code_slug <- NULL
} else {
vendor_address_zip_code_slug <- df_row$vendor_address_zip_code
}
if (is.na(df_row$reason_for_modification)) {
reason_for_modification_slug <- NULL
} else {
reason_for_modification_slug <- df_row$reason_for_modification
}
if (is.na(current_contract_value)) {
current_contract_value_slug <- NULL
} else {
current_contract_value_slug <- df_row$current_contract_value
}
if (is.na(dollars_obligated)) {
dollars_obligated_slug <- NULL
} else {
dollars_obligated_slug <- df_row$dollars_obligated
}
if (is.na(contract_value)) {
contract_value_slug <- NULL
} else {
contract_value_slug <- df_row$contract_value
}
if (is.na(fee_range_lower_value)) {
fee_range_lower_value_slug <- NULL
} else {
fee_range_lower_value_slug <- df_row$fee_range_lower_value
}
if (is.na(fee_range_upper_value)) {
fee_range_upper_value_slug <- NULL
} else {
}
if (is.na(fixed_fee_value)) {
fixed_fee_value_slug <- NULL
} else {
fixed_fee_value_slug <- df_row$fixed_fee_value
}
if (is.na(total_current_contract_value)) {
total_current_contract_value_slug <- NULL
} else {
total_current_contract_value_slug <-
df_row$total_current_contract_value
}
if (is.na(total_dollars_obligated)) {
total_dollars_obligated_slug <- NULL
} else {
total_dollars_obligated_slug <- df_row$total_dollars_obligated
}
if (is.na(total_non_government_value)) {
total_non_government_value_slug <- NULL
} else {
total_non_government_value_slug <- df_row$total_non_government_value
}
if (is.na(total_ultimate_contract_value)) {
total_ultimate_contract_value_slug <- NULL
} else {
total_ultimate_contract_value_slug <-
df_row$total_ultimate_contract_value
}
if (is.na(contract_fiscal_year)) {
contract_fiscal_year_slug <- NULL
} else {
contract_fiscal_year_slug <- df_row$contract_fiscal_year
}
if (is.na(extent_competed)) {
extent_competed_slug <- NULL
} else {
extent_competed_slug <- df_row$extent_competed
}
if (is.na(number_of_offers_received)) {
number_of_offers_received_slug <- NULL
} else {
number_of_offers_received_slug <- df_row$number_of_offers_received
}
d <-
.fpds_single_csv_search_safe(
fpds_type = fpds_type,
vendor_doing_business_as_name = vendor_doing_business_as_name_slug,
award_type = award_type_slug,
global_vendor_name = global_vendor_name_slug,
vendor_name = vendor_name_slug,
parent_vendor_name = parent_vendor_name_slug,
vendor_duns_number = vendor_duns_number_slug,
parent_duns_number = parent_duns_number_slug,
agency_name = agency_name_slug,
contracting_office_name = contracting_office_name_slug,
contracting_agency_name = contracting_agency_name_slug,
principal_naics_code = principal_naics_code_slug,
award_status = award_status_slug,
contract_type = contract_type_slug,
contract_type_description = contract_type_description_slug,
type_of_contract_pricing = type_of_contract_pricing_slug,
contract_id = contract_id_slug,
ref_idv_contract_id = ref_idv_contract_id_slug,
ref_idv_agency_id = ref_idv_agency_id_slug,
contracting_agency_id = contracting_agency_id_slug,
contracting_office_id = contracting_office_id_slug,
funding_agency_id = funding_agency_id_slug,
funding_office_id = funding_office_id_slug,
funding_office_name = funding_office_name_slug,
agency_code = agency_code_slug,
department_id = department_id_slug,
department_name = department_name_slug,
research = research_slug,
last_mod_date = last_mod_date_slug,
last_modified_by = last_modified_by_slug,
award_completion_date = award_completion_date_slug,
created_date = created_date_slug,
signed_date = signed_date_slug,
effective_date = effective_date_slug,
estimated_completion_date = estimated_completion_date_slug,
obligated_amount = obligated_amount_slug,
base_exercised_options_value = base_exercised_options_value_slug,
ultimate_contract_value = ultimate_contract_value_slug,
created_by = created_by_slug,
description_of_requirement = description_of_requirement_slug,
reason_for_modification = reason_for_modification_slug,
legislative_mandates = legislative_mandates_slug,
local_area_set_aside = local_area_set_aside_slug,
socio_economic_indicators = socio_economic_indicators_slug,
multiyear_contract = multiyear_contract_slug,
national_interest_code = national_interest_code_slug,
product_or_service_code = product_or_service_code_slug,
performance_district_code = performance_district_code_slug,
performance_country = performance_country_slug,
performance_state_name = performance_state_name_slug,
vendor_address_city = vendor_address_city_slug,
vendor_congress_district_code = vendor_congress_district_code_slug,
vendor_address_country_code = vendor_address_country_code_slug,
vendor_address_country_name = vendor_address_country_name_slug,
vendor_address_state_code = vendor_address_state_code_slug,
vendor_address_state_name = vendor_address_state_name_slug,
vendor_address_zip_code = vendor_address_zip_code_slug,
sort_item = sort_item,
sort_descending = sort_descending,
return_message = return_message,
cancellation_date = cancellation_date_slug,
destroy_date = destroy_date_slug,
final_invoice_paid_date = final_invoice_paid_date_slug,
funded_through_date = funded_through_date_slug,
last_modified_date = last_modified_date_slug,
physical_completion_date = physical_completion_date_slug,
reveal_date = reveal_date_slug,
solicitation_issue_date = solicitation_issue_date_slug,
sys_last_modified_date = sys_last_modified_date_slug,
vendor_registration_date = vendor_registration_date_slug,
vendor_renewal_date = vendor_renewal_date_slug,
solicitation_procedure = solicitation_procedure_slug,
subcontract_plan = subcontract_plan_slug,
total_dollars_obligated = total_dollars_obligated_slug,
current_contract_value = current_contract_value_slug,
total_current_contract_value = total_current_contract_value_slug,
total_ultimate_contract_value = total_ultimate_contract_value_slug,
contract_fiscal_year = contract_fiscal_year_slug,
...
)
if (d %>% hasName("codeModification")) {
d <- d %>%
mutate(codeModification = as.character(codeModification))
}
if (d %>% tibble::has_name("idAgencyAward")) {
d <- d %>%
mutate(idAgencyAward = as.character(idAgencyAward))
}
if (!d %>% tibble::has_name("zipcodeVendor")) {
d <- d %>%
mutate(zipcodeVendor = NA_character_)
}
if (d %>% hasName("codeProductService")) {
d <- d %>%
mutate(codeProductService = as.character(codeProductService))
}
d <- d %>%
mutate(zipcodeVendor = as.character(zipcodeVendor))
d
})
}
if (nrow(all_data) == 0) {
message("No results")
return(invisible())
}
all_data <-
all_data %>%
.munge_data(clean_address = F, exclude_bloat = exclude_bloat)
df_psc <- dictionary_psc_active()
all_data <-
all_data %>%
left_join(
df_psc %>% select(typePSC,
codeProductService,
nameProductService) %>% mutate_all(str_to_upper)
,
by = "codeProductService"
) %>%
distinct() %>%
select(-one_of("descriptionProductService"))
all_data <-
all_data %>%
.remove_na()
if (all_data %>% hasName("codeProductService")) {
all_data <- all_data %>%
mutate(
idSolicitationGroup =
case_when(
is.na(codeProductService) ~ NA_character_,
codeProductService %>% substr(1, 1) %>% str_detect("[A-Z]") ~ codeProductService %>% substr(1, 1),
TRUE ~ codeProductService %>% substr(1, 2)
)
)
all_data <- all_data %>%
left_join(df_psc %>% distinct(idSolicitationGroup, nameSolicitationGroup),
by = "idSolicitationGroup")
}
if (all_data %>% hasName("nameVendorParent") &&
all_data %>% hasName("nameVendorParent")) {
all_data <-
all_data %>%
mutate(
nameVendor = case_when(
is.na(nameVendor) & !is.na(nameVendorParent) ~ nameVendorParent,
is.na(nameVendor) &
is.na(nameVendor) ~ "UNDISCLOSED VENDOR",
TRUE ~ nameVendor
),
nameVendorParent = case_when(
is.na(nameVendorParent) & !is.na(nameVendor) ~ nameVendorParent,
is.na(nameVendor) &
is.na(nameVendor) ~ "UNDISCLOSED VENDOR PARENT",
TRUE ~ nameVendorParent
)
)
}
if (all_data %>% hasName("dateContractSigned")) {
all_data <-
all_data %>%
mutate(dateObligation = dateContractSigned)
}
if (all_data %>% hasName("idDUNS") &
all_data %>% hasName("idDUNSParent")) {
all_data <-
all_data %>%
mutate_at(c("idDUNS", "idDUNSParent"),
list(function(x) {
ifelse(x == 0, NA_integer_, x)
})) %>%
mutate(hasParent = idDUNSParent != idDUNS)
}
if (all_data %>% hasName("idContractIDV")) {
all_data <- all_data %>%
mutate(isIDV = !is.na(idContractIDV))
}
if (clean_entity_column) {
all_data <-
.clean_org_usa(data = all_data,
col = "nameAgencyAward",
clean_entity_column = clean_entity_column)
all_data <-
.clean_org_usa(data = all_data,
col = "nameOfficeAward",
clean_entity_column = clean_entity_column)
all_data <-
.clean_organization(data = all_data,
clean_entity_column = clean_entity_column,
col = "nameVendor")
all_data <-
.clean_organization(data = all_data,
clean_entity_column = clean_entity_column,
col = "nameVendorParent")
}
if (all_data %>% hasName("yearObligation")) {
all_data <- all_data %>%
rename(yearFiscalObligation = yearObligation)
}
all_data <-
all_data %>%
select(-matches("slugDUNS|dateContractSigned")) %>%
select(one_of(c("idContract", "dateObligation")), everything())
if (clean_entity_column) {
all_data <-
.clean_organization(data = all_data, col = "nameVendor")
all_data <-
.clean_organization(data = all_data, col = "nameVendorParent")
}
if (unformat) {
all_data <-
all_data %>%
mutate_if(is.numeric, as.numeric)
}
all_data <-
all_data %>%
arrange(dateObligation)
if (decode_contract_ids) {
all_data <-
tbl_decode_federal_contract_ids(data = all_data, use_sam = F)
}
if (snake_names) {
all_data <-
all_data %>%
janitor::clean_names()
}
all_data
}
#' Clean FPDS Agencies
#'
#' Cleans malformed FPDS
#' Agencies and brings in parents.
#'
#' @param data a \code{tibble}
#'
#' @return
#' @export
#'
#' @examples
clean_fpds_agencies <-
function(data) {
dict_agencies <- dictionary_government_agencies()
all_names <- names(data)
if (data %>% hasName("nameAgencyAward")) {
df_distinct <-
data %>%
filter(!is.na(idAgencyAward)) %>%
distinct(idAgencyAward, nameAgencyAward)
df_distinct <-
df_distinct %>%
left_join(
dict_agencies %>%
select(
nameAgencyAwardParent = nameAgency,
idAgencyAward = idOffice,
nameAgencyAwardNew = nameOffice
),
by = "idAgencyAward"
) %>%
group_by(idAgencyAward) %>%
dplyr::slice(1) %>%
ungroup()
df_distinct <- df_distinct %>%
mutate(nameAgencyAward = ifelse(
is.na(nameAgencyAwardNew),
nameAgencyAward,
nameAgencyAwardNew
)) %>%
select(-nameAgencyAwardNew)
data <-
data %>% select(-nameAgencyAward) %>%
left_join(df_distinct, by = "idAgencyAward") %>%
select(one_of(names(data)), everything())
}
if (data %>% hasName("nameAgencyAward")) {
df_distinct <-
data %>%
filter(!is.na(idAgencyAward)) %>%
distinct(idAgencyAward, nameAgencyAward)
df_distinct <-
df_distinct %>%
left_join(
dict_agencies %>%
select(
nameAgencyAwardParent = nameAgency,
idAgencyAward = idOffice,
nameAgencyAwardNew = nameOffice
),
by = "idAgencyAward"
) %>%
group_by(idAgencyAward) %>%
dplyr::slice(1) %>%
ungroup()
df_distinct <-
df_distinct %>%
mutate(nameAgencyAward = ifelse(
is.na(nameAgencyAwardNew),
nameAgencyAward,
nameAgencyAwardNew
)) %>%
select(-nameAgencyAwardNew)
data <-
data %>% select(-nameAgencyAward) %>%
left_join(df_distinct, by = "idAgencyAward") %>%
select(one_of(names(data)), everything())
}
if (data %>% hasName("nameAgencyFunding")) {
df_distinct <-
data %>%
filter(!is.na(idAgencyFunding)) %>%
distinct(idAgencyFunding, nameAgencyFunding)
df_distinct <-
df_distinct %>%
left_join(
dict_agencies %>%
select(
nameAgencyFundingParent = nameAgency,
idAgencyFunding = idOffice,
nameAgencyFundingNew = nameOffice
),
by = "idAgencyFunding"
) %>%
group_by(idAgencyFunding) %>%
dplyr::slice(1) %>%
ungroup()
df_distinct <-
df_distinct %>%
mutate(nameAgencyFunding = ifelse(
is.na(nameAgencyFundingNew),
nameAgencyFunding,
nameAgencyFundingNew
)) %>%
select(-nameAgencyFundingNew)
data <-
data %>% select(-nameAgencyFunding) %>%
left_join(df_distinct, by = "idAgencyFunding") %>%
select(one_of(names(data)), everything())
}
if (data %>% hasName("nameAgencyAwardIDV")) {
df_distinct <-
data %>%
filter(!is.na(idAgencyAwardIDV)) %>%
distinct(idAgencyAwardIDV, nameAgencyAwardIDV)
df_distinct <-
df_distinct %>%
left_join(
dict_agencies %>%
select(
nameAgencyAwardIDVParent = nameAgency,
idAgencyAwardIDV = idOffice,
nameAgencyAwardIDVNew = nameOffice
),
by = "idAgencyAwardIDV"
) %>%
group_by(idAgencyAwardIDV) %>%
dplyr::slice(1) %>%
ungroup()
df_distinct <-
df_distinct %>%
mutate(nameAgencyAwardIDV = ifelse(
is.na(nameAgencyAwardIDVNew),
nameAgencyAwardIDV,
nameAgencyAwardIDVNew
)) %>%
select(-nameAgencyAwardIDVNew)
data <-
data %>% select(-nameAgencyAwardIDV) %>%
left_join(df_distinct, by = "idAgencyAwardIDV") %>%
select(one_of(names(data)), everything())
}
data
}
.parse_atom_xml_simple <-
function(url) {
doc <- read_xml(url)
xml_nodes <-
doc %>%
xml_contents()
df <- tibble(
column = xml_nodes %>% as_list() %>% unlist() %>% names() %>% str_remove_all(
"content.award.|content.IDV.|awardID.|relevantContractDates.|dollarValues.|totalDollarValues.|purchaserInformation.|contractData.|contractMarketingData.|ccrRegistrationDetails.|legislativeMandates.|vendor.vendorSiteDetails.|productOrServiceInformation.|listOfTreasuryAccounts.treasuryAccount.|vendorSocioEconomicIndicators.| vendor.vendorHeader.|vendorLineOfBusiness.|vendorBusinessTypes.|minorityOwned.|preferencePrograms.|placeOfPerformance.principalPlaceOfPerformance.|competition.|vendorRelationshipWithFederalGovernment.|typeOfGovernmentEntity.|vendorOrganizationFactors.|vendorCertifications.|treasuryAccountSymbol.|vendor.vendorHeader.|localGovernment.|federalGovernment.|businessOrOrganizationType.|profitStructure.|typeOfEducationalEntity.| placeOfPerformance.|contentOtherTransactionIDVcontractDetail.|\\contentIDVtransactionInformation|\\contractID"
),
value = xml_nodes %>% as_list() %>% unlist() %>% as.character()
)
df <-
df %>%
filter(column != "") %>%
mutate(idColumn = 1:n()) %>%
mutate(column = column %>% str_remove_all("\\."))
start_col <- df %>% slice(1) %>% select(column) %>% pull()
df_ids <-
df %>% filter(column == start_col) %>%
mutate(numberTransaction = 1:n()) %>%
select(numberTransaction, everything())
df <-
df %>%
left_join(df_ids %>% select(idColumn, numberTransaction)
, by = "idColumn") %>%
fill(numberTransaction) %>%
select(numberTransaction, everything())
data <-
df %>%
mutate(
column = column %>% str_remove_all(
"contractID| contentOtherTransactionIDVcontractDetail|contentOtherTransactionIDV| ^contractDetail"
)
) %>%
group_by(numberTransaction, idColumn, column) %>%
filter(idColumn == min(idColumn)) %>%
ungroup() %>%
select(numberTransaction, column, value)
col_order <- data$column %>% unique()
data <- data %>%
spread(column, value) %>%
select(one_of(col_order))
if (data %>%
hasName("vendorDUNSInformationvendorName")) {
data <-
data %>% rename(nameVendorDUNS = vendorDUNSInformationvendorName)
}
names(data) <-
names(data) %>% str_remove_all("contentOtherTransactionAward|contentOtherTransactionIDV|^contractID")
data <-
data %>%
dplyr::select(-matches("^generic"))
data <- .munge_fpds_names(data)
col_ids <-
tibble(name = names(data)) %>%
mutate(column = 1:n()) %>%
group_by(name) %>%
filter(column == min(column)) %>%
ungroup() %>%
pull(column)
data <- data[, col_ids]
data <- data %>%
.fix_contract_description() %>%
.fix_contract_name()
data <-
data %>%
mutate(urlFPDSContractAtom = url)
data
}
#' FPDS Special Search
#'
#' @param ...
#'
#' @return
#' @export
#'
#' @examples
fpds_s <-
function(snake_names = F,
unformat = T,
clean_entity_column = F,
exclude_bloat = F,
munge_tbl = T,
return_message = T,
...) {
d <- fpds_atom(...) %>% nrow() * 10
if (d < 15000) {
data <-
.fpds_single_csv_search(return_message = return_message, ...)
} else {
df_dates <- .generate_search_dates()
data <-
1:nrow(df_dates) %>%
map_dfr(function(x) {
df_date <- df_dates %>% dplyr::slice(x)
.fpds_csv_safe <-
possibly(.fpds_single_csv_search, tibble())
data <-
.fpds_csv_safe(
signed_date = c(df_date$start_date, df_date$end_date),
return_message = return_message,
...
)
}) %>%
mutate_if(is.numeric, as.numeric)
}
if (clean_entity_column) {
data <-
.clean_organization(data = data, col = "nameVendor")
data <-
.clean_organization(data = data, col = "nameVendorParent")
}
if (data %>% hasName("dateContractSigned")) {
data <-
data %>%
rename(dateObligation = dateContractSigned)
}
if (!munge_tbl) {
return(data)
}
data <-
data %>%
arrange(dateObligation)
data <-
data %>%
munge_lite(
snake_names = snake_names,
unformat = unformat,
exclude_bloat = exclude_bloat
)
data
}
#' Parse atom URLS simple form
#'
#' @param urls vector of urls
#'
#' @return
#' @export
#'
#' @examples
parse_fpds_atom_urls_simple <-
function(urls, exclude_bloat = F) {
urls %>%
future_map_dfr(function(x) {
.parse_atom_xml_simple(url = x)
}) %>%
.munge_data(exclude_bloat = exclude_bloat)
}
# xml ---------------------------------------------------------------------
.parse_content_nodes_detailed <-
function(content_nodes) {
all_data <-
seq_along(content_nodes) %>%
map_dfr(function(y) {
content_node <- content_nodes[[y]]
items <- names(as_list(content_node) %>% unlist())
values <-
as_list(content_node) %>% unlist() %>% as.character()
column <- xml_name(content_node)
#column %>% message()
double_nodes <-
c(
"awardID",
"contractID",
"agencySpecificAwardElements",
"OtherTransactionAwardID"
)
if (column %in% double_nodes) {
items <-
content_node %>% xml_contents() %>% xml_contents()
if (column == "OtherTransactionAwardID") {
column <- "awardID"
}
values <-
items %>% xml_text()
}
single_nodes <-
c(
"relevantContractDates",
"dollarValues",
"totalDollarValues",
"purchaserInformation",
"contractMarketingData",
"contractData",
"legislativeMandates",
"productOrServiceInformation",
"preferencePrograms",
"competition",
"transactionInformation",
"genericTags"
)
if (column %in% c("vendor", "contractDetail")) {
values <-
content_node %>% xml_contents() %>% .[[2]] %>% xml_contents() %>% xml_contents() %>% xml_text()
items <-
content_node %>% xml_contents() %>% .[[2]] %>% xml_contents() %>% xml_contents() %>% xml_name()
descriptions <-
content_node %>% xml_contents() %>% .[[2]] %>% xml_contents() %>% xml_contents() %>% xml_attrs() %>% as.character()
df <-
tibble(nameItem = items,
value = values,
description = descriptions)
items_2 <-
content_node %>% xml_contents() %>% xml_contents() %>% xml_contents() %>%
xml_contents() %>%
xml_name()
values_2 <-
content_node %>% xml_contents() %>% xml_contents() %>% xml_contents() %>%
xml_contents() %>%
xml_text()
descriptions_2 <-
content_node %>% xml_contents() %>% xml_contents() %>%
xml_contents() %>%
xml_contents() %>% xml_attrs() %>% as.character()
df_append <-
tibble(nameItem = items_2,
value = values_2,
description = descriptions_2) %>%
filter(nameItem != "text")
if (column == "contractDetail") {
column <- "vendor"
}
df <- df %>%
bind_rows(df_append) %>%
mutate(numberNode = y,
nameNode = column) %>%
select(numberNode, nameNode, everything())
return(df)
}
if (column == "placeOfPerformance") {
cn <- content_node %>% xml_contents()
df <-
seq_along(cn) %>%
map_dfr(function(x) {
values <-
cn[[x]] %>% xml_children() %>% xml_text()
if (length(values) == 0) {
items <- cn[[x]] %>% xml_name()
values <- cn[[x]] %>% xml_text()
} else {
items <-
cn[[x]] %>% xml_children() %>% xml_name()
}
tibble(nameItem = items, value = values)
})
df <-
df %>%
mutate(
description = "character(0)",
numberNode = y,
nameNode = column
) %>%
select(numberNode, nameNode, everything())
return(df)
}
if (!column %in% double_nodes) {
items <-
content_node %>% xml_contents()
values <-
items %>% xml_text()
}
item_names <- items %>% map(xml_name) %>% flatten_chr()
item_attributes <- items %>% map(xml_attrs)
descriptions <-
seq_along(item_attributes) %>%
map(function(x) {
values <-
item_attributes[[x]] %>% as.character() %>% str_c(collapse = " | ")
if (length(values) == 0) {
return(NA_character_)
}
values %>% gsub("\\s+", " ", .)
}) %>%
flatten_chr()
df_items <-
tibble(nameItem = item_names, description = descriptions)
df_items %>%
mutate(
value = str_trim(values),
description = descriptions,
numberNode = y,
nameNode = column
) %>%
select(numberNode, nameNode, everything())
})
all_data <-
all_data %>%
mutate(description = ifelse(description == "character(0)", NA, description)) %>%
mutate(idRow = 1:n())
df_values <-
all_data %>%
mutate(nameItem = ifelse(nameItem == nameNode, nameItem, str_c(nameNode, nameItem))) %>%
mutate(nameItem = nameItem %>% str_remove_all("\\-")) %>%
select(nameItem, value) %>%
filter(value != "")
df_order <-
df_values %>%
mutate(idRow = 1:n()) %>%
select(idRow, nameItem)
df_values <-
df_values %>%
select(nameItem, value) %>%
group_by(nameItem) %>%
dplyr::slice(1) %>%
ungroup() %>%
spread(nameItem, value)
df_descriptions <-
all_data %>% filter(!is.na(description)) %>%
mutate(nameItem = ifelse(
nameItem == nameNode,
nameItem,
str_c(nameNode, nameItem, "Description")
)) %>%
mutate(nameItem = nameItem %>% str_remove_all("\\-"))
df_descriptions <-
df_descriptions %>%
select(idRow, nameItem, description) %>%
filter(!is.na(description))
df_order <-
df_order %>%
bind_rows(df_descriptions %>% select(idRow, nameItem)) %>%
arrange(idRow)
df_descriptions <-
df_descriptions %>%
select(-idRow) %>%
group_by(nameItem) %>%
dplyr::slice(1) %>%
spread(nameItem, description)
data <-
df_values %>% bind_cols(df_descriptions) %>%
select(one_of(df_order$nameItem), everything()) %>%
mutate_if(is.character,
list(function(x) {
x %>% gsub("\\s+", " ", .)
})) %>%
mutate_if(is.numeric, as.numeric)
data
}
.parse_xml_entry <- function(xml_entry, use_detailed = F) {
xml_entry_nodes <- xml_contents(xml_entry)
xml_entry_names <- xml_name(xml_entry_nodes)
seq_along(xml_entry_names) %>%
map(function(x) {
table <-
xml_entry_names[[x]]
if (table == "title") {
title <- xml_entry_nodes[1] %>% xml_text()
df <- tibble(nameContract = title)
return(df)
}
if (table == "link") {
url <-
xml_entry_nodes[[x]] %>% html_attr("href")
df <- tibble(urlFPDSContractAtom = url)
return(df)
}
if (table == "modified") {
value <-
xml_entry_nodes[x] %>% xml_text()
if (value == "") {
return(invisible())
}
value <- value %>% lubridate::ymd_hms()
df <- tibble(datetimeContractModified = value)
return(df)
}
xml_part <- xml_entry_nodes[x]
values <- xml_part %>% as_list()
value <- values %>% unlist() %>% as.character()
items <- values %>% unlist() %>% names()
items <-
seq_along(items) %>%
map_chr(function(x) {
parts <-
items[[x]] %>%
str_split("\\.") %>% flatten_chr() %>%
str_remove_all("NASASpecificAwardElements")
start <- length(parts) - 1
end <- length(parts)
parts <-
parts[start:end] %>% str_c(collapse = "")
parts %>% substr(1, 1) %>% str_to_lower() %>%
str_c(.,
parts %>% substr(2, nchar(parts)), collapse = "")
})
df <-
tibble(item = items, value)
df <- df %>% select(item, value)
df <-
df %>% left_join(dictionary_fpds_names() %>% rename(item = nameFPDS),
by = "item")
if (df %>% filter(is.na(nameActual)) %>% nrow() > 0) {
missing_items <-
df %>% filter(is.na(nameActual)) %>% pull(item)
seq_along(missing_items) %>%
map(function(x) {
glue("Missing {missing_items[[x]]}") %>% message()
})
df <- df %>%
mutate(nameActual = case_when(is.na(nameActual) ~ item,
TRUE ~ nameActual))
}
df <-
df %>%
select(nameActual, value)
df <-
df %>%
group_by(nameActual) %>%
dplyr::slice(1) %>%
ungroup()
col_order <- df$nameActual
df %>%
spread(nameActual, value) %>% select(one_of(col_order)) %>%
mutate_if(is.numeric, as.numeric)
}) %>% reduce(bind_cols)
}
.parse_fpds_atom_url <-
function(url = "https://www.fpds.gov/ezsearch/FEEDS/ATOM?FEEDNAME=PUBLIC&q=GLOBAL_VENDOR_NAME:PALANTIR&sortBy=SIGNED_DATE&desc=Y&start=520&sortBy=SIGNED_DATE",
keep_key_columns = F,
return_message = T) {
if (return_message) {
message(str_c("Parsing: ", url))
}
doc <-
read_xml(url)
xml_nodes <-
xml_contents(doc)
tables <-
xml_name(xml_nodes)
df_table_nodes <-
tibble(table = tables) %>%
mutate(idTable = 1:n())
df_ids <-
filter(.data = df_table_nodes, table == "entry")
data <-
seq_len(nrow(df_ids)) %>%
map_dfr(function(node_x) {
df_row <-
df_ids[node_x, ]
table <- df_row$table
idTable <- df_row$idTable
if (table == "title") {
title <-
xml_contents(doc) %>% xml_text() %>% .[[1]] %>% str_to_upper() %>% gsub("\\s+", " ", .)
df <- tibble(titleSearch = title)
return(df)
}
if (table == "link") {
url <-
xml_nodes[[idTable]] %>% html_attr("href")
df <- tibble(urlFPDSAtom = url)
return(df)
}
if (table == "modified") {
value <- xml_nodes[[idTable]] %>% xml_text()
if (value == "") {
return(invisible())
}
}
if (table == "author") {
value <- xml_nodes[[idTable]] %>% xml_text()
if (value == "") {
return(invisible())
}
}
xml_entry <-
xml_nodes[[idTable]]
df <-
.parse_xml_entry_new(xml_entry = xml_entry)
if (df %>% hasName("idContractIDV") &
df %>% hasName("idAgencyAward")) {
award <- unique(df$idAgencyAward) %>% str_c(collapse = "|")
award <- glue("^{award}") %>% as.character()
df <- df %>%
mutate(idContractIDV = idContractIDV %>% str_remove_all(award))
}
if (df %>% hasName("codeModificationIDV") &&
df %>% hasName("idContractIDV")) {
idvmod <-
unique(df$codeModificationIDV) %>% str_c(collapse = "|")
idvmod <- as.character(glue("{idvmod}$"))
df <- df %>%
mutate(idContractIDV = idContractIDV %>% str_remove_all(idvmod))
}
df
})
df_double <-
tibble(name = names(data)) %>% count(name, sort = T) %>% filter(n > 1)
if (nrow(df_double) > 0) {
df_double$name %>% str_c(collapse = '\n') %>% message()
}
data <-
select(.data = data, -matches("remove"))
if (!data %>%
hasName("dateContractSigned") &
data %>% hasName("datetimeContractSigned")) {
data <-
data %>%
rename(dateContractSigned = datetimeContractSigned)
}
start_1 <-
data %>%
select(
matches(
"idDUNS|nameVendor|nameAgency|nameOffice|descriptionOfficeFundingRequest|descriptionObligation|acronymProgram|nameMajorProgram"
)
) %>% names()
start_2 <-
data %>%
select(matches("amount|^count[A-Z]")) %>%
select(-matches("country")) %>%
names()
one_names <- data %>% select(ends_with("1")) %>% names()
if (length(one_names) > 0) {
data <- data %>% select(-one_of(one_names))
}
sel_col <- names(data)[names(data) %in% c(
"nameContract",
"idContract",
"dateContractSigned",
"dateContractEffective",
"idAgencyFunding"
)]
data <-
data %>%
mutate(urlFPDSAtom = url) %>%
select(one_of(sel_col),
matches("idDUNS"),
one_of(c(start_1, start_2)),
everything())
if (data %>% hasName("genericString06")) {
data <- data %>%
select(-genericString06)
}
if (keep_key_columns) {
data <-
select_key_columns(data)
}
data
}
.parse_xml_fpds_new <- function(url = "https://www.fpds.gov/ezsearch/FEEDS/ATOM?FEEDNAME=PUBLIC&q=GLOBAL_VENDOR_NAME:PALANTIR&sortBy=SIGNED_DATE&desc=Y&start=520&sortBy=SIGNED_DATE", keep_key_columns = F, return_message = F) {
if (return_message) {
message(str_c("Parsing: ", url))
}
doc <- read_xml(url)
xml_ns_strip(doc)
entries <- doc %>% xml_find_all("//entry")
data <-
seq_along(entries) %>%
map_dfr(function(x){
.parse_xml_entry_new(xml_entry = entries[[x]])
})
df_double <-
tibble(name = names(data)) %>% count(name, sort = T) %>% filter(n > 1)
if (nrow(df_double) > 0) {
df_double$name %>% str_c(collapse = '\n') %>% message()
}
data <-
select(.data = data, -matches("remove"))
if (!data %>%
hasName("dateContractSigned") &
data %>% hasName("datetimeContractSigned")) {
data <-
data %>%
rename(dateContractSigned = datetimeContractSigned)
}
start_1 <-
data %>%
select(
matches(
"idDUNS|nameVendor|nameAgency|nameOffice|descriptionOfficeFundingRequest|descriptionObligation|acronymProgram|nameMajorProgram"
)
) %>% names()
start_2 <-
data %>%
select(matches("amount|^count[A-Z]")) %>%
select(-matches("country")) %>%
names()
one_names <- data %>% select(ends_with("1")) %>% names()
if (length(one_names) > 0) {
data <- data %>% select(-one_of(one_names))
}
sel_col <- names(data)[names(data) %in% c(
"nameContract",
"idContract",
"dateContractSigned",
"dateContractEffective",
"idAgencyFunding"
)]
data <-
data %>%
mutate(urlFPDSAtom = url) %>%
select(one_of(sel_col),
matches("idDUNS"),
one_of(c(start_1, start_2)),
everything())
if (keep_key_columns) {
data <-
select_key_columns(data)
}
data
}
.parse_fpds_atom_urls_multi <-
function(urls = c(
"http://api.gdeltproject.org/api/v2/doc/doc?query=brooklyn%20nets%20sourcelang:english&mode=artlist&format=json×pan=12w&maxrecords=250&sort=datedesc",
"http://api.gdeltproject.org/api/v2/doc/doc?query=domain:netsdaily.com%20sourcelang:english&mode=artlist&format=json×pan=12w&maxrecords=250&sort=datedesc",
"http://api.gdeltproject.org/api/v2/doc/doc?query=theme:econ_bitcoin%20sourcelang:english&mode=artlist&format=json×pan=12w&maxrecords=250&sort=datedesc"
),
snake_names = T,
clean_entity_column = F,
unformat = T,
exclude_bloat = T,
clean_address = T,
return_message = T,
keep_key_columns = TRUE) {
all_data <-
tibble()
.parse_xml_fpds_new_safe <-
purrr::possibly(.parse_xml_fpds_new, tibble())
success <- function(res) {
data <-
res$url %>%
.parse_xml_fpds_new_safe(keep_key_columns = keep_key_columns)
all_data <<-
all_data %>%
bind_rows(data)
}
failure <- function(msg) {
tibble()
}
urls %>%
walk(function(x) {
curl_fetch_multi(url = x, success, failure)
})
multi_run()
if (return_message) {
message("\n\nData acquired\n\n")
}
if (all_data %>% hasName("zipcodePerformancePerformance") &&
all_data %>% hasName("zipcodePerformance")) {
all_data <- all_data %>%
mutate(
zipcodePerformance = case_when(
is.na(zipcodePerformance) ~ zipcodePerformancePerformance,
TRUE ~ zipcodePerformance
)
) %>%
select(-zipcodePerformancePerformance)
}
if (all_data %>% hasName("nameVendorParent") &
all_data %>% hasName("nameVendor")) {
all_data <- all_data %>%
mutate(
nameVendor = case_when(
is.na(nameVendor) & !is.na(nameVendorParent) ~ nameVendorParent,
is.na(nameVendor) &
is.na(nameVendor) ~ "UNDISCLOSED VENDOR",
TRUE ~ nameVendor
),
nameVendorParent = case_when(
is.na(nameVendorParent) & !is.na(nameVendor) ~ nameVendorParent,
is.na(nameVendor) &
is.na(nameVendor) ~ "UNDISCLOSED VENDOR PARENT",
TRUE ~ nameVendorParent
)
) %>%
mutate_at(c("idDUNS", "idDUNSParent"),
list(function(x) {
ifelse(x == 0, NA_integer_, x)
})) %>%
mutate(hasContractParent = idDUNSParent != idDUNS) %>%
mutate_if(is.numeric, as.numeric)
}
if (all_data %>% hasName("nameVendorParent") &
!all_data %>% hasName("nameVendor")) {
all_data <- all_data %>%
mutate(nameVendor = nameVendorParent)
}
if (all_data %>% hasName("idContractIDV")) {
all_data <- all_data %>%
mutate(isIDV = !is.na(idContractIDV))
}
if (all_data %>% hasName("idCGAC") &
all_data %>% hasName("codeAccountOMB")) {
all_data <-
.generate_federal_account_ids(data = all_data,
cgac_column = "idCGAC",
account_column = "codeAccountOMB")
}
all_data <-
all_data %>%
.fix_contract_name() %>%
.fix_contract_description()
names(all_data) <-
names(all_data) %>% str_replace_all("PerformancePerformance", "Performance")
if (all_data %>% hasName("zipcodePerformance") &
all_data %>% hasName("locationPerformance")) {
all_data <- all_data %>%
mutate(zipcodePerformance = case_when(
is.na(zipcodePerformance) ~ locationPerformance,
TRUE ~ zipcodePerformance
)) %>%
select(-locationPerformance)
}
change_v <- all_data %>% hasName("nameContractor") &&
all_data %>% hasName("nameVendor")
if (change_v) {
all_data <- all_data %>%
mutate(isContractorNotVendor = nameContractor != nameVendor)
}
if (all_data %>% hasName("dateContractSigned")) {
all_data <-
mutate(all_data, dateObligation = dateContractSigned %>% as.Date())
} else {
if (all_data %>% hasName("datetimeTransactionCreated")) {
all_data <-
all_data %>%
mutate(dateObligation = as.Date(datetimeTransactionCreated))
}
if (all_data %>% hasName("datetimeContractModified")) {
all_data <- all_data %>%
mutate(dateObligation = as.Date(datetimeContractModified))
}
}
all_data <-
select(
all_data,
one_of("dateObligation"),
matches("idContract"),
matches("dateContractEffective"),
one_of("nameContract"),
matches("description"),
matches("datetimeContractModified"),
matches("idDUNS|Vendor"),
matches("date"),
matches("amount"),
matches("Department"),
matches("Agency"),
matches("Office"),
matches("NAICS"),
matches("ProductService"),
everything()
)
if (all_data %>% hasName("dateContractEffective") &&
all_data %>% hasName("dateObligation")) {
all_data <- all_data %>%
mutate(
countDaysSinceContractEffective = (dateObligation - dateContractEffective) %>% as.numeric(),
isFutureObligation = countDaysSinceContractEffective < 0
)
}
if (all_data %>% hasName("dateContractCompletionCurrent") &
all_data %>% hasName("dateContractCompletionUltimate")) {
all_data <- all_data %>%
mutate(hasModifiedContractDate = dateContractCompletionUltimate != dateContractCompletionCurrent)
}
dt <-
all_data %>% select(matches("datetimeReg|datetimeRen")) %>% names()
if (length(dt) > 0) {
all_data <- mutate_at(all_data, dt, as.Date)
new <-
names(all_data)[names(all_data) %in% dt] %>% str_remove_all("time")
names(all_data)[names(all_data) %in% dt] <- new
}
if (clean_entity_column) {
all_data <-
.clean_org_usa(data = all_data,
col = "nameAgencyAward",
clean_entity_column = clean_entity_column)
all_data <-
.clean_org_usa(data = all_data,
col = "nameAgencyFunding",
clean_entity_column = clean_entity_column)
all_data <-
.clean_org_usa(data = all_data,
col = "nameOfficeAward",
clean_entity_column = clean_entity_column)
all_data <-
.clean_org_usa(data = all_data,
col = "nameOfficeFunding",
clean_entity_column = clean_entity_column)
all_data <-
.clean_organization(data = all_data,
clean_entity_column = clean_entity_column,
col = "nameVendor")
all_data <-
.clean_organization(data = all_data,
clean_entity_column = clean_entity_column,
col = "nameContractor")
all_data <-
.clean_organization(data = all_data,
clean_entity_column = clean_entity_column,
col = "nameVendorParent")
}
all_data <- mutate_if(all_data, is.numeric, as.numeric)
all_data <-
munge_data(
data = all_data,
unformat = unformat,
exclude_bloat = exclude_bloat,
clean_address = clean_address,
snake_names = F
)
if (all_data %>% hasName("idContractAnalysis") &&
all_data %>% hasName("idContractIDV")) {
all_data <- all_data %>%
mutate(idContractAnalysis = case_when(
is.na(idContractAnalysis) ~ idContractIDV,
TRUE ~ idContractAnalysis
))
}
if (all_data %>% hasName("idContract") &
all_data %>% hasName("idAward")) {
all_data <-
all_data %>%
mutate(idContract = ifelse(is.na(idContract), idAward, idContract)) %>%
select(-idAward)
}
if ((!all_data %>% hasName("idContract")) &
(all_data %>% hasName("idAward"))) {
all_data <-
all_data %>%
rename(idContract = idAward)
}
all_data <-
unique(all_data) %>%
.remove_na()
if (!all_data %>% hasName("nameNAICS") &
all_data %>% hasName("idNAICS")) {
df_naics <-
all_data %>%
select(idNAICS) %>%
filter(!is.na(idNAICS)) %>%
distinct()
df_naics <- df_naics %>%
left_join(dictionary_naics_codes(),
by = "idNAICS")
all_data <-
all_data %>%
select(-nameNAICS) %>%
left_join(df_naics, by = "idNAICS")
df_naics_names <-
all_data %>%
group_by(idNAICS, nameNAICS) %>%
summarise(amount = sum(amountObligation, na.rm = T)) %>%
filter(amount == max(amount)) %>%
dplyr::slice(1) %>%
select(idNAICS, nameNAICS) %>%
ungroup()
all_data <-
all_data %>%
select(-nameNAICS) %>%
left_join(df_naics_names, by = "idNAICS") %>%
select(names(all_data), everything())
}
if (!all_data %>% hasName("nameProductService") &
all_data %>% hasName("codeProductService")) {
df_psc <-
all_data %>%
filter(!is.na(codeProductService)) %>%
distinct(codeProductService)
df_psc <-
df_psc %>%
left_join(
dictionary_psc_active() %>% select(typePSC,
codeProductService,
nameProductService) %>% mutate_all(str_to_upper)
,
by = "codeProductService"
)
all_data <-
all_data %>%
select(-one_of("nameProductService")) %>%
left_join(df_psc, by = "codeProductService")
all_data <- add_solicitation_group(data = all_data)
df_psc_names <-
all_data %>%
group_by(codeProductService, nameProductService) %>%
summarise(amount = sum(amountObligation, na.rm = T)) %>%
filter(amount == max(amount)) %>%
dplyr::slice(1) %>%
select(codeProductService, nameProductService) %>%
ungroup()
all_data <- all_data %>%
select(-nameProductService) %>%
left_join(df_psc_names, by = "codeProductService") %>%
select(names(all_data), everything())
}
if (all_data %>% hasName("nameVendor") &
all_data %>% hasName("nameVendorParent")) {
all_data <- all_data %>%
mutate(nameVendor = case_when(is.na(nameVendor) ~ nameVendor,
TRUE ~ nameVendorParent))
}
if (all_data %>% hasName("typeResearch")) {
all_data <- all_data %>%
mutate(
isPossibleSBIRSTTR = descriptionObligation %>% str_detect(
"(PHASE 1)|(PHASE I)|(PHASE II)|(PHASE 2)|(PHASE III)|STTR|SBIR"
),
isFilteredSBIR = case_when(
typeResearch %in%
c(
"SMALL BUSINESS INNOVATION RESEARCH PROGRAM PHASE I ACTION",
"SMALL BUSINESS INNOVATION RESEARCH PROGRAM PHASE II ACTION",
"SMALL TECHNOLOGY TRANSFER RESEARCH PROGRAM PHASE I",
"SMALL TECHNOLOGY TRANSFER RESEARCH PROGRAM PHASE II"
) ~ T,
is.na(typeResearch) & isPossibleSBIRSTTR ~ T,
TRUE ~ F
)
)
}
if (all_data %>% hasName("nameFirstInvestigator") &
all_data %>% hasName("nameLastInvestigator")) {
df_pis <-
all_data %>%
distinct(nameFirstInvestigator, nameLastInvestigator) %>%
filter(!is.na(nameFirstInvestigator)) %>%
filter(!nameFirstInvestigator %>% str_detect("UNKNOWN")) %>%
filter(!is.na(nameLastInvestigator) %>% str_detect("UNKNOWN"))
if (nrow(df_pis) > 0) {
df_pis <-
df_pis %>%
mutate(
namePrincipalInvestigator = case_when(
nameFirstInvestigator %>% str_detect("UNKNOWN") ~ NA_character_,
TRUE ~ str_c(nameFirstInvestigator, nameLastInvestigator, sep = " ")
)
) %>%
filter(!is.na(namePrincipalInvestigator))
df_pis <- df_pis %>%
clean_entity_data(entity_column = "namePrincipalInvestigator") %>%
select(-namePrincipalInvestigator) %>%
rename(namePrincipalInvestigator = namePrincipalInvestigatorClean)
all_data <-
all_data %>%
left_join(df_pis,
by = c("nameFirstInvestigator", "nameLastInvestigator"))
}
}
## ENTITY RESOLUTION ### NEEDS FIXING
if (all_data %>% hasName("cageVendor")) {
df_cage <-
all_data %>%
select(idDUNSParent, cageVendor, amountObligation) %>%
filter(!is.na(cageVendor)) %>%
count(idDUNSParent, cageVendor, wt = amountObligation) %>%
group_by(cageVendor) %>%
filter(n == max(n)) %>%
slice(1) %>%
ungroup() %>%
select(idDUNSParent, cageVendor)
all_data <-
all_data %>%
select(-cageVendor) %>%
left_join(df_cage, by = "idDUNSParent") %>%
distinct() %>%
select(one_of(names(all_data)), everything())
}
if (all_data %>% hasName("typeAward")) {
all_data <- all_data %>%
mutate(
typeProcurementFPDS = case_when(
typeAward %>% str_detect("BPA") ~ "BLANKET PURCHASE AGREEMENT",
TRUE ~ typeAward
),
isGrantType = typeAward %>% str_detect("GRANT")
)
}
if (all_data %>% hasName("yearObligation")) {
all_data <- all_data %>%
rename(yearFiscalObligation = yearObligation)
}
all_data <-
all_data %>%
select(-matches("slugDUNS|dateContractSigned")) %>%
select(one_of(c(
"idContractAnalysis", "idContract", "dateObligation"
)), everything())
name_cols <-
all_data %>% select(matches("nameAgency|nameOffice|nameDepartment")) %>% names()
name_cols %>%
walk(function(x) {
all_data <<- .clean_org_usa(data = all_data,
col = x,
clean_entity_column = clean_entity_column)
})
if (exclude_bloat) {
all_data <- all_data %>%
select(-matches("urlFPDS"))
}
all_data <- arrange(.data = all_data, dateObligation)
all_data
}
#' Parse ATOM FPDS urls
#'
#' @param urls vector of fpds URLS
#' @param return_message if \code{TRUE} return a message
#' @param clean_entity_column if \code{TRUE} cleans entity columns
#' @param unformat if \code{TRUE} unformats formattable columns
#' @param keep_key_columns if \code{TRUE} returns only key columns
#' @param exclude_bloat
#' @param snake_names
#' @param use_future
#' @param clean_address
#' @param show_progress
#'
#' @return \code{tibble}
#' @export
#'
#' @examples
#' @example inst/examples/parse_atom_urls.r
parse_fpds_atom_urls <-
function(urls = NULL,
use_future = T,
clean_entity_column = F,
unformat = T,
keep_key_columns = F,
exclude_bloat = T,
clean_address = T,
snake_names = F,
show_progress = T,
return_message = T) {
options(warn = -1)
if (length(urls) == 0) {
stop("Enter ATOM URLs")
}
.parse_fpds_atom_url_safe <-
possibly(.parse_fpds_atom_urls_multi, tibble())
if (use_future) {
options(future.globals.maxSize = 999999 * 1024 ^ 12)
all_data <-
urls %>%
future_map_dfr(function(url) {
.parse_fpds_atom_url_safe(urls = url,
snake_names = F,
clean_entity_column = clean_entity_column,
unformat = unformat,
exclude_bloat = exclude_bloat,
clean_address = clean_address,
return_message = F,
keep_key_columns = keep_key_columns)
})
} else {
all_data <-
urls %>%
map_dfr(function(url) {
.parse_fpds_atom_url_safe(urls = url,
snake_names = F,
clean_entity_column = clean_entity_column,
unformat = unformat,
exclude_bloat = exclude_bloat,
clean_address = clean_address,
return_message = F,
keep_key_columns = keep_key_columns)
})
}
if (nrow(all_data) == 0) {
"No data" %>% message()
return(tibble())
}
actions <- nrow(all_data) %>% comma(digits = 0)
contracts <-
all_data %>% distinct(idContractAnalysis) %>% nrow() %>% comma(digits = 0)
duns <-
all_data %>% distinct(idDUNS) %>% nrow() %>% comma(digits = 0)
parent_duns <-
all_data %>% distinct(idDUNSParent) %>% nrow() %>% comma(digits = 0)
amt <-
all_data$amountObligation %>% sum() %>% currency(digits = 0)
from_date <- all_data$dateObligation %>% min(na.rm = T)
to_date <- all_data$dateObligation %>% max(na.rm = T)
glue(
"\n\n{green({amt})} procured between {red({from_date})} and {red({to_date})} across {yellow(actions)} actions amongst {magenta(contracts)} contracts allocated to {cyan(duns)} distinct DUNS and {blue(parent_duns)} distinct parent DUNS\n\n"
) %>% message()
all_data <- arrange(.data = all_data, dateObligation)
if (snake_names) {
all_data <-
all_data %>%
clean_names()
}
all_data
}
parse_fpds_atom_urls_old <-
function(urls = NULL,
use_future = T,
clean_entity_column = F,
unformat = T,
keep_key_columns = F,
exclude_bloat = T,
clean_address = T,
snake_names = F,
show_progress = T,
return_message = T) {
options(warn = -1)
if (length(urls) == 0) {
stop("Enter ATOM URLs")
}
.parse_fpds_atom_url_safe <-
possibly(parse_fpds_atom_urls_multi, tibble())
if (use_future) {
options(future.globals.maxSize = 999999 * 1024 ^ 12)
all_data <-
urls %>%
future_map_dfr(function(url) {
.parse_fpds_atom_url_safe(
url = url,
keep_key_columns = keep_key_columns,
return_message = return_message
)
}, .progress = show_progress)
} else {
all_data <-
urls %>%
map_dfr(function(url) {
.parse_fpds_atom_url_safe(
url = url,
keep_key_columns = keep_key_columns,
return_message = return_message
)
})
}
if (nrow(all_data) == 0) {
"No data" %>% message()
return(tibble())
}
if (return_message) {
message("\n\nData acquired\n\n")
}
if (all_data %>% hasName("zipcodePerformancePerformance") &&
all_data %>% hasName("zipcodePerformance")) {
all_data <- all_data %>%
mutate(
zipcodePerformance = case_when(
is.na(zipcodePerformance) ~ zipcodePerformancePerformance,
TRUE ~ zipcodePerformance
)
) %>%
select(-zipcodePerformancePerformance)
}
if (all_data %>% hasName("nameVendorParent") &
all_data %>% hasName("nameVendor")) {
all_data <- all_data %>%
mutate(
nameVendor = case_when(
is.na(nameVendor) & !is.na(nameVendorParent) ~ nameVendorParent,
is.na(nameVendor) &
is.na(nameVendor) ~ "UNDISCLOSED VENDOR",
TRUE ~ nameVendor
),
nameVendorParent = case_when(
is.na(nameVendorParent) & !is.na(nameVendor) ~ nameVendorParent,
is.na(nameVendor) &
is.na(nameVendor) ~ "UNDISCLOSED VENDOR PARENT",
TRUE ~ nameVendorParent
)
) %>%
mutate_at(c("idDUNS", "idDUNSParent"),
list(function(x) {
ifelse(x == 0, NA_integer_, x)
})) %>%
mutate(hasContractParent = idDUNSParent != idDUNS) %>%
mutate_if(is.numeric, as.numeric)
}
if (all_data %>% hasName("nameVendorParent") &
!all_data %>% hasName("nameVendor")) {
all_data <- all_data %>%
mutate(nameVendor = nameVendorParent)
}
if (all_data %>% hasName("idContractIDV")) {
all_data <- all_data %>%
mutate(isIDV = !is.na(idContractIDV))
}
if (all_data %>% hasName("idCGAC") &
all_data %>% hasName("codeAccountOMB")) {
all_data <-
.generate_federal_account_ids(data = all_data,
cgac_column = "idCGAC",
account_column = "codeAccountOMB")
}
all_data <-
all_data %>%
.fix_contract_name() %>%
.fix_contract_description()
names(all_data) <-
names(all_data) %>% str_replace_all("PerformancePerformance", "Performance")
if (all_data %>% hasName("zipcodePerformance") &
all_data %>% hasName("locationPerformance")) {
all_data <- all_data %>%
mutate(zipcodePerformance = case_when(
is.na(zipcodePerformance) ~ locationPerformance,
TRUE ~ zipcodePerformance
)) %>%
select(-locationPerformance)
}
change_v <- all_data %>% hasName("nameContractor") &&
all_data %>% hasName("nameVendor")
if (change_v) {
all_data <- all_data %>%
mutate(isContractorNotVendor = nameContractor != nameVendor)
}
if (all_data %>% hasName("dateContractSigned")) {
all_data <- mutate(all_data, dateObligation = dateContractSigned %>% as.Date())
} else {
if (all_data %>% hasName("datetimeTransactionCreated")) {
all_data <-
all_data %>%
mutate(dateObligation = as.Date(datetimeTransactionCreated))
}
if (all_data %>% hasName("datetimeContractModified")) {
all_data <- all_data %>%
mutate(dateObligation = as.Date(datetimeContractModified))
}
}
all_data <-
select(
all_data,
one_of("dateObligation"),
matches("idContract"),
matches("dateContractEffective"),
one_of("nameContract"),
matches("description"),
matches("datetimeContractModified"),
matches("idDUNS|Vendor"),
matches("date"),
matches("amount"),
matches("Department"),
matches("Agency"),
matches("Office"),
matches("NAICS"),
matches("ProductService"),
everything()
)
if (all_data %>% hasName("dateContractEffective") && all_data %>% hasName("dateObligation")) {
all_data <- all_data %>%
mutate(
countDaysSinceContractEffective = (dateObligation - dateContractEffective) %>% as.numeric(),
isFutureObligation = countDaysSinceContractEffective < 0
)
}
if (all_data %>% hasName("dateContractCompletionCurrent") &
all_data %>% hasName("dateContractCompletionUltimate")) {
all_data <- all_data %>%
mutate(hasModifiedContractDate = dateContractCompletionUltimate != dateContractCompletionCurrent)
}
dt <-
all_data %>% select(matches("datetimeReg|datetimeRen")) %>% names()
if (length(dt) > 0) {
all_data <- mutate_at(all_data, dt, as.Date)
new <-
names(all_data)[names(all_data) %in% dt] %>% str_remove_all("time")
names(all_data)[names(all_data) %in% dt] <- new
}
if (clean_entity_column) {
all_data <-
.clean_org_usa(data = all_data,
col = "nameAgencyAward",
clean_entity_column = clean_entity_column)
all_data <-
.clean_org_usa(data = all_data,
col = "nameAgencyFunding",
clean_entity_column = clean_entity_column)
all_data <-
.clean_org_usa(data = all_data,
col = "nameOfficeAward",
clean_entity_column = clean_entity_column)
all_data <-
.clean_org_usa(data = all_data,
col = "nameOfficeFunding",
clean_entity_column = clean_entity_column)
all_data <-
.clean_organization(data = all_data,
clean_entity_column = clean_entity_column,
col = "nameVendor")
all_data <-
.clean_organization(data = all_data,
clean_entity_column = clean_entity_column,
col = "nameContractor")
all_data <-
.clean_organization(data = all_data,
clean_entity_column = clean_entity_column,
col = "nameVendorParent")
}
all_data <- mutate_if(all_data, is.numeric, as.numeric)
all_data <-
munge_data(
data = all_data,
unformat = unformat,
exclude_bloat = exclude_bloat,
clean_address = clean_address,
snake_names = F
)
if (all_data %>% hasName("idContractAnalysis") &&
all_data %>% hasName("idContractIDV")) {
all_data <- all_data %>%
mutate(idContractAnalysis = case_when(
is.na(idContractAnalysis) ~ idContractIDV,
TRUE ~ idContractAnalysis
))
}
if (all_data %>% hasName("idContract") &
all_data %>% hasName("idAward")) {
all_data <-
all_data %>%
mutate(idContract = ifelse(is.na(idContract), idAward, idContract)) %>%
select(-idAward)
}
if ((!all_data %>% hasName("idContract")) &
(all_data %>% hasName("idAward"))) {
all_data <-
all_data %>%
rename(idContract = idAward)
}
all_data <-
unique(all_data) %>%
.remove_na()
if (!all_data %>% hasName("nameNAICS") &
all_data %>% hasName("idNAICS")) {
df_naics <-
all_data %>%
select(idNAICS) %>%
filter(!is.na(idNAICS)) %>%
distinct()
df_naics <- df_naics %>%
left_join(dictionary_naics_codes(),
by = "idNAICS")
all_data <-
all_data %>%
select(-nameNAICS) %>%
left_join(df_naics, by = "idNAICS")
df_naics_names <-
all_data %>%
group_by(idNAICS, nameNAICS) %>%
summarise(amount = sum(amountObligation, na.rm = T)) %>%
filter(amount == max(amount)) %>%
dplyr::slice(1) %>%
select(idNAICS, nameNAICS) %>%
ungroup()
all_data <-
all_data %>%
select(-nameNAICS) %>%
left_join(df_naics_names, by = "idNAICS") %>%
select(names(all_data), everything())
}
if (!all_data %>% hasName("nameProductService") &
all_data %>% hasName("codeProductService")) {
df_psc <-
all_data %>%
filter(!is.na(codeProductService)) %>%
distinct(codeProductService)
df_psc <-
df_psc %>%
left_join(
dictionary_psc_active() %>% select(typePSC,
codeProductService,
nameProductService) %>% mutate_all(str_to_upper)
,
by = "codeProductService"
)
all_data <-
all_data %>%
select(-one_of("nameProductService")) %>%
left_join(df_psc, by = "codeProductService")
all_data <- add_solicitation_group(data = all_data)
df_psc_names <-
all_data %>%
group_by(codeProductService, nameProductService) %>%
summarise(amount = sum(amountObligation, na.rm = T)) %>%
filter(amount == max(amount)) %>%
dplyr::slice(1) %>%
select(codeProductService, nameProductService) %>%
ungroup()
all_data <- all_data %>%
select(-nameProductService) %>%
left_join(df_psc_names, by = "codeProductService") %>%
select(names(all_data), everything())
}
if (all_data %>% hasName("nameVendor") &
all_data %>% hasName("nameVendorParent")) {
all_data <- all_data %>%
mutate(nameVendor = case_when(is.na(nameVendor) ~ nameVendor,
TRUE ~ nameVendorParent))
}
if (all_data %>% hasName("typeResearch")) {
all_data <- all_data %>%
mutate(
isPossibleSBIRSTTR = descriptionObligation %>% str_detect(
"(PHASE 1)|(PHASE I)|(PHASE II)|(PHASE 2)|(PHASE III)|STTR|SBIR"
),
isFilteredSBIR = case_when(
typeResearch %in%
c(
"SMALL BUSINESS INNOVATION RESEARCH PROGRAM PHASE I ACTION",
"SMALL BUSINESS INNOVATION RESEARCH PROGRAM PHASE II ACTION",
"SMALL TECHNOLOGY TRANSFER RESEARCH PROGRAM PHASE I",
"SMALL TECHNOLOGY TRANSFER RESEARCH PROGRAM PHASE II"
) ~ T,
is.na(typeResearch) & isPossibleSBIRSTTR ~ T,
TRUE ~ F
)
)
}
if (all_data %>% hasName("nameFirstInvestigator") &
all_data %>% hasName("nameLastInvestigator")) {
df_pis <-
all_data %>%
distinct(nameFirstInvestigator, nameLastInvestigator) %>%
filter(!is.na(nameFirstInvestigator)) %>%
filter(!nameFirstInvestigator %>% str_detect("UNKNOWN")) %>%
filter(!is.na(nameLastInvestigator) %>% str_detect("UNKNOWN"))
if (nrow(df_pis) > 0) {
df_pis <-
df_pis %>%
mutate(
namePrincipalInvestigator = case_when(
nameFirstInvestigator %>% str_detect("UNKNOWN") ~ NA_character_,
TRUE ~ str_c(nameFirstInvestigator, nameLastInvestigator, sep = " ")
)
) %>%
filter(!is.na(namePrincipalInvestigator))
df_pis <- df_pis %>%
clean_entity_data(entity_column = "namePrincipalInvestigator") %>%
select(-namePrincipalInvestigator) %>%
rename(namePrincipalInvestigator = namePrincipalInvestigatorClean)
all_data <-
all_data %>%
left_join(df_pis,
by = c("nameFirstInvestigator", "nameLastInvestigator"))
}
}
## ENTITY RESOLUTION ### NEEDS FIXING
if (all_data %>% hasName("cageVendor")) {
df_cage <-
all_data %>%
select(idDUNSParent, cageVendor, amountObligation) %>%
filter(!is.na(cageVendor)) %>%
count(idDUNSParent, cageVendor, wt = amountObligation) %>%
group_by(cageVendor) %>%
filter(n == max(n)) %>%
slice(1) %>%
ungroup() %>%
select(idDUNSParent, cageVendor)
all_data <-
all_data %>%
select(-cageVendor) %>%
left_join(df_cage, by = "idDUNSParent") %>%
distinct() %>%
select(one_of(names(all_data)), everything())
}
if (all_data %>% hasName("typeAward")) {
all_data <- all_data %>%
mutate(
typeProcurementFPDS = case_when(
typeAward %>% str_detect("BPA") ~ "BLANKET PURCHASE AGREEMENT",
TRUE ~ typeAward
),
isGrantType = typeAward %>% str_detect("GRANT")
)
}
if (all_data %>% hasName("yearObligation")) {
all_data <- all_data %>%
rename(yearFiscalObligation = yearObligation)
}
all_data <-
all_data %>%
select(-matches("slugDUNS|dateContractSigned")) %>%
select(one_of(c(
"idContractAnalysis", "idContract", "dateObligation"
)), everything())
name_cols <-
all_data %>% select(matches("nameAgency|nameOffice|nameDepartment")) %>% names()
name_cols %>%
walk(function(x) {
all_data <<- .clean_org_usa(data = all_data,
col = x,
clean_entity_column = clean_entity_column)
})
if (exclude_bloat) {
all_data <- all_data %>%
select(-matches("urlFPDS"))
}
actions <- nrow(all_data) %>% comma(digits = 0)
contracts <-
all_data %>% distinct(idContractAnalysis) %>% nrow() %>% comma(digits = 0)
duns <-
all_data %>% distinct(idDUNS) %>% nrow() %>% comma(digits = 0)
parent_duns <-
all_data %>% distinct(idDUNSParent) %>% nrow() %>% comma(digits = 0)
amt <-
all_data$amountObligation %>% sum() %>% currency(digits = 0)
from_date <- all_data$dateObligation %>% min(na.rm = T)
to_date <- all_data$dateObligation %>% max(na.rm = T)
glue(
"\n\n{green({amt})} procured between {red({from_date})} and {red({to_date})} across {yellow(actions)} actions amongst {magenta(contracts)} contracts allocated to {cyan(duns)} distinct DUNS and {blue(parent_duns)} distinct parent DUNS\n\n"
) %>% message()
all_data <- arrange(.data = all_data, dateObligation)
if (snake_names) {
all_data <-
all_data %>%
clean_names()
}
all_data
}
# generate ----------------------------------------------------------------
.generate_atom_urls <-
function(url = "https://fpds.gov/ezsearch/FEEDS/ATOM?s=FPDS&FEEDNAME=PUBLIC&VERSION=1.5.1&q=VENDOR_DUNS_NUMBER:036593457") {
doc <- read_xml(url)
xml_nodes <-
xml_contents(x = doc)
urls <- xml_nodes %>% html_attr("href")
items <- xml_nodes %>% html_attr("rel")
df_url <-
tibble(item = items, url = urls) %>% filter(item == "last")
if (nrow(df_url) == 0) {
return(tibble(urlFPDSAtom = url))
}
total_records <-
df_url$url %>% str_split("\\&start=") %>% flatten_chr() %>% .[[2]] %>% as.numeric()
times <- total_records %/% 10 + 1
seq_ids <- seq(0, by = 10, length.out = times)
urls <-
glue::glue("{url}&start={seq_ids}&sortBy=SIGNED_DATE") %>% as.character() %>% map_chr(URLencode)
data <- tibble(idSequence = 1:times, urlFPDSAtom = urls)
data
}
.generate_base_api_call <-
function(vendor_doing_business_as_name = NULL,
global_vendor_name = NULL,
award_type = NULL,
vendor_name = NULL,
solicitation_procedure = NULL,
vendor_duns_number = NULL,
agency_name = NULL,
contracting_office_name = NULL,
contracting_agency_name = NULL,
principal_naics_code = NULL,
award_status = NULL,
contract_type = NULL,
contract_type_description = NULL,
type_of_contract_pricing = NULL,
contract_id = NULL,
contracting_agency_id = NULL,
contracting_office_id = NULL,
funding_agency_id = NULL,
funding_office_id = NULL,
funding_office_name = NULL,
agency_code = NULL,
department_id = NULL,
subcontract_plan = NULL,
department_name = NULL,
last_mod_date = NULL,
last_modified_by = NULL,
award_completion_date = NULL,
created_date = NULL,
signed_date = NULL,
effective_date = NULL,
estimated_completion_date = NULL,
ref_idv_contract_id = NULL,
cancellation_date = NULL,
destroy_date = NULL,
final_invoice_paid_date = NULL,
funded_through_date = NULL,
last_modified_date = NULL,
physical_completion_date = NULL,
reveal_date = NULL,
solicitation_issue_date = NULL,
sys_last_modified_date = NULL,
vendor_registration_date = NULL,
vendor_renewal_date = NULL,
number_of_offers_received = NULL,
base_exercised_options_value = NULL,
current_contract_value = NULL,
dollars_obligated = NULL,
contract_value = NULL,
fee_range_lower_value = NULL,
fee_range_upper_value = NULL,
fixed_fee_value = NULL,
obligated_amount = NULL,
total_current_contract_value = NULL,
total_dollars_obligated = NULL,
total_non_government_value = NULL,
total_ultimate_contract_value = NULL,
ultimate_contract_value = NULL,
contract_fiscal_year = NULL,
description_of_requirement = NULL,
reason_for_modification = NULL,
legislative_mandates = NULL,
local_area_set_aside = NULL,
socio_economic_indicators = NULL,
multiyear_contract = NULL,
national_interest_code = NULL,
product_or_service_code = NULL,
performance_district_code = NULL,
performance_country = NULL,
performance_state_name = NULL,
vendor_address_city = NULL,
vendor_congress_district_code = NULL,
vendor_address_country_code = NULL,
vendor_address_country_name = NULL,
vendor_address_state_code = NULL,
vendor_address_state_name = NULL,
vendor_address_zip_code = NULL,
extent_competed = NULL,
research = NULL,
...) {
options(scipen = 999999)
params = list(
piid = contract_id,
global_vendor_name = global_vendor_name,
award_type = award_type,
solicitation_proc_description = solicitation_procedure,
subcontract_plan = subcontract_plan,
contracting_office_id = contracting_office_id,
contract_type_description = contract_type_description,
contracting_office_name = contracting_office_name,
funding_agency_id = funding_agency_id,
funding_office_id = funding_office_id,
funding_office_name = funding_office_name,
agency_code = agency_code,
agency_name = agency_name,
department_id = department_id,
department_name = department_name,
last_mod_date = last_mod_date,
last_modified_by = last_modified_by,
award_completion_date = award_completion_date,
cancellation_date = cancellation_date,
destroy_date = destroy_date,
final_invoice_paid_date = final_invoice_paid_date,
funded_through_date = funded_through_date,
last_modified_date = last_modified_date,
physical_completion_date = physical_completion_date,
reveal_date = reveal_date,
solicitation_issue_date = solicitation_issue_date,
sys_last_modified_date = sys_last_modified_date,
vendor_registration_date = vendor_registration_date,
vendor_renewal_date = vendor_renewal_date,
created_date = created_date,
signed_date = signed_date,
effective_date = effective_date,
estimated_completion_date = estimated_completion_date,
number_of_offers_received = number_of_offers_received,
base_exercised_options_value = base_exercised_options_value,
current_contract_value = current_contract_value,
dollars_obligated = dollars_obligated,
contract_value = contract_value,
fee_range_lower_value = fee_range_lower_value,
fee_range_upper_value = fee_range_upper_value,
fixed_fee_value = fixed_fee_value,
obligated_amount = obligated_amount,
total_current_contract_value = total_current_contract_value,
total_dollars_obligated = total_dollars_obligated,
total_non_government_value = total_non_government_value,
total_ultimate_contract_value = total_ultimate_contract_value,
ultimate_contract_value = ultimate_contract_value,
contract_fiscal_year = contract_fiscal_year,
type_of_contract_pricing = type_of_contract_pricing,
award_status = award_status,
contract_type = contract_type,
description_of_requirement = description_of_requirement,
reason_for_modification = reason_for_modification,
legislative_mandates = legislative_mandates,
local_area_set_aside = local_area_set_aside,
socio_economic_indicators = socio_economic_indicators,
multiyear_contract = multiyear_contract,
national_interest_code = national_interest_code,
principal_naics_code = principal_naics_code,
product_or_service_code = product_or_service_code,
pop_congress_district_code = performance_district_code,
pop_country_name = performance_country,
pop_state_name = performance_state_name,
vendor_address_city = vendor_address_city,
vendor_congress_district_code = vendor_congress_district_code,
vendor_address_country_code = vendor_address_country_code,
vendor_address_country_name = vendor_address_country_name,
vendor_duns_number = vendor_duns_number,
vendor_doing_business_as_name = vendor_doing_business_as_name,
vendor_name = vendor_name,
vendor_address_state_code = vendor_address_state_code,
vendor_address_state_name = vendor_address_state_name,
vendor_address_zip_code = vendor_address_zip_code,
research = research,
ref_idv_contract_id = ref_idv_contract_id,
contracting_agency_name = contracting_agency_name,
contracting_office_id = contracting_office_id
)
params <-
utils::modifyList(params, list(...))
params
}
.fpds_api_call_params <-
function(vendor_doing_business_as_name = NULL,
global_vendor_name = NULL,
vendor_name = NULL,
vendor_duns_number = NULL,
award_type = NULL,
solicitation_procedure = NULL,
agency_name = NULL,
contracting_office_name = NULL,
contracting_agency_name = NULL,
principal_naics_code = NULL,
award_status = NULL,
contract_type = NULL,
contract_type_description = NULL,
type_of_contract_pricing = NULL,
contract_id = NULL,
ref_idv_contract_id = NULL,
ref_idv_agency_id = NULL,
modification_number = NULL,
contracting_agency_id = NULL,
contracting_office_id = NULL,
subcontract_plan = NULL,
funding_agency_id = NULL,
funding_office_id = NULL,
funding_office_name = NULL,
agency_code = NULL,
department_id = NULL,
department_name = NULL,
last_mod_date = NULL,
last_modified_by = NULL,
award_completion_date = NULL,
created_date = NULL,
signed_date = NULL,
effective_date = NULL,
estimated_completion_date = NULL,
cancellation_date = NULL,
destroy_date = NULL,
final_invoice_paid_date = NULL,
funded_through_date = NULL,
last_modified_date = NULL,
physical_completion_date = NULL,
reveal_date = NULL,
solicitation_issue_date = NULL,
sys_last_modified_date = NULL,
vendor_registration_date = NULL,
vendor_renewal_date = NULL,
base_exercised_options_value = NULL,
current_contract_value = NULL,
dollars_obligated = NULL,
contract_value = NULL,
fee_range_lower_value = NULL,
fee_range_upper_value = NULL,
fixed_fee_value = NULL,
obligated_amount = NULL,
total_current_contract_value = NULL,
total_dollars_obligated = NULL,
total_non_government_value = NULL,
total_ultimate_contract_value = NULL,
ultimate_contract_value = NULL,
contract_fiscal_year = NULL,
created_by = NULL,
description_of_requirement = NULL,
reason_for_modification = NULL,
legislative_mandates = NULL,
research = NULL,
extent_competed = NULL,
local_area_set_aside = NULL,
socio_economic_indicators = NULL,
multiyear_contract = NULL,
national_interest_code = NULL,
national_interest_description = NULL,
naics_description = NULL,
product_or_service_code = NULL,
product_or_service_description = NULL,
performance_district_code = NULL,
performance_country = NULL,
performance_state_name = NULL,
vendor_address_city = NULL,
vendor_congress_district_code = NULL,
vendor_address_country_code = NULL,
vendor_address_country_name = NULL,
vendor_address_state_code = NULL,
vendor_address_state_name = NULL,
vendor_address_zip_code = NULL,
number_of_offers_received = NULL,
return_query = F,
...) {
result_list <-
.generate_base_api_call(
piid = contract_id,
global_vendor_name = global_vendor_name,
award_type = award_type,
modification_number = modification_number,
contracting_agency_id = contracting_agency_id,
contracting_agency_name = contracting_agency_name,
contracting_office_id = contracting_office_id,
contract_type_description = contract_type_description,
contracting_office_name = contracting_office_name,
funding_agency_id = funding_agency_id,
funding_office_id = funding_office_id,
funding_office_name = funding_office_name,
agency_code = agency_code,
agency_name = agency_name,
department_id = department_id,
department_name = department_name,
last_mod_date = last_mod_date,
last_modified_by = last_modified_by,
award_completion_date = award_completion_date,
subcontract_plan = subcontract_plan,
created_date = created_date,
signed_date = signed_date,
effective_date = effective_date,
estimated_completion_date = estimated_completion_date,
number_of_offers_received = number_of_offers_received,
base_exercised_options_value = base_exercised_options_value,
current_contract_value = current_contract_value,
dollars_obligated = dollars_obligated,
contract_value = contract_value,
fee_range_lower_value = fee_range_lower_value,
fee_range_upper_value = fee_range_upper_value,
fixed_fee_value = fixed_fee_value,
obligated_amount = obligated_amount,
total_current_contract_value = total_current_contract_value,
total_dollars_obligated = total_dollars_obligated,
total_non_government_value = total_non_government_value,
total_ultimate_contract_value = total_ultimate_contract_value,
ultimate_contract_value = ultimate_contract_value,
contract_fiscal_year = contract_fiscal_year,
type_of_contract_pricing = type_of_contract_pricing,
award_status = award_status,
contract_type = contract_type,
description_of_requirement = description_of_requirement,
reason_for_modification = reason_for_modification,
legislative_mandates = legislative_mandates,
local_area_set_aside = local_area_set_aside,
socio_economic_indicators = socio_economic_indicators,
multiyear_contract = multiyear_contract,
national_interest_code = national_interest_code,
national_interest_description = national_interest_description,
principal_naics_code = principal_naics_code,
naics_description = naics_description,
product_or_service_code = product_or_service_code,
product_or_service_description = product_or_service_description,
pop_congress_district_code = performance_district_code,
pop_country_name = performance_country,
pop_state_name = performance_state_name,
vendor_address_city = vendor_address_city,
vendor_congress_district_code = vendor_congress_district_code,
vendor_address_country_code = vendor_address_country_code,
vendor_address_country_name = vendor_address_country_name,
vendor_duns_number = vendor_duns_number,
vendor_doing_business_as_name = vendor_doing_business_as_name,
vendor_name = vendor_name,
vendor_address_state_code = vendor_address_state_code,
vendor_address_state_name = vendor_address_state_name,
vendor_address_zip_code = vendor_address_zip_code,
contract_id = contract_id,
performance_district_code = performance_district_code,
performance_country = performance_country,
performance_state_name = performance_state_name,
extent_competed = extent_competed,
research = research,
cancellation_date = cancellation_date,
destroy_date = destroy_date,
final_invoice_paid_date = final_invoice_paid_date,
funded_through_date = funded_through_date,
last_modified_date = last_modified_date,
physical_completion_date = physical_completion_date,
reveal_date = reveal_date,
solicitation_issue_date = solicitation_issue_date,
sys_last_modified_date = sys_last_modified_date,
vendor_registration_date = vendor_registration_date,
vendor_renewal_date = vendor_renewal_date,
solicitation_procedure = solicitation_procedure,
ref_idv_contract_id = ref_idv_contract_id,
...
)
if (result_list %>% map_dbl(length) %>% sum() == 0) {
stop("No parameters")
}
df_parameters <-
result_list %>% .munge_double_list() %>%
.create_fpds_query_params()
if (return_query) {
df_values <-
df_parameters %>%
gather(param, value)
query <-
df_values %>%
mutate(
value = value %>% str_to_upper() %>% map_chr(URLencode),
param = str_to_upper(param)
) %>%
unite(slug, param, value, sep = ":") %>%
pull(slug) %>%
str_c(collapse = "%20") %>%
map_chr(URLencode)
return(query)
}
df_parameters
}
.generate_fpds_advanced_query <-
function(vendor_doing_business_as_name = NULL,
award_type = NULL,
global_vendor_name = NULL,
vendor_name = NULL,
vendor_duns_number = NULL,
agency_name = NULL,
contracting_office_name = NULL,
solicitation_procedure = NULL,
contracting_agency_name = NULL,
principal_naics_code = NULL,
award_status = NULL,
contract_type = NULL,
contract_type_description = NULL,
type_of_contract_pricing = NULL,
contract_id = NULL,
ref_idv_contract_id = NULL,
ref_idv_agency_id = NULL,
modification_number = NULL,
contracting_agency_id = NULL,
contracting_office_id = NULL,
funding_agency_id = NULL,
funding_office_id = NULL,
funding_office_name = NULL,
subcontract_plan = NULL,
agency_code = NULL,
department_id = NULL,
department_name = NULL,
last_mod_date = NULL,
last_modified_by = NULL,
award_completion_date = NULL,
created_date = NULL,
signed_date = NULL,
effective_date = NULL,
estimated_completion_date = NULL,
cancellation_date = NULL,
destroy_date = NULL,
final_invoice_paid_date = NULL,
funded_through_date = NULL,
last_modified_date = NULL,
physical_completion_date = NULL,
reveal_date = NULL,
solicitation_issue_date = NULL,
sys_last_modified_date = NULL,
vendor_registration_date = NULL,
vendor_renewal_date = NULL,
base_exercised_options_value = NULL,
current_contract_value = NULL,
dollars_obligated = NULL,
contract_value = NULL,
fee_range_lower_value = NULL,
fee_range_upper_value = NULL,
fixed_fee_value = NULL,
obligated_amount = NULL,
total_current_contract_value = NULL,
total_dollars_obligated = NULL,
total_non_government_value = NULL,
total_ultimate_contract_value = NULL,
ultimate_contract_value = NULL,
contract_fiscal_year = NULL,
created_by = NULL,
description_of_requirement = NULL,
reason_for_modification = NULL,
legislative_mandates = NULL,
local_area_set_aside = NULL,
socio_economic_indicators = NULL,
multiyear_contract = NULL,
national_interest_code = NULL,
national_interest_description = NULL,
naics_description = NULL,
product_or_service_code = NULL,
product_or_service_description = NULL,
performance_district_code = NULL,
performance_country = NULL,
performance_state_name = NULL,
research = NULL,
vendor_address_city = NULL,
vendor_congress_district_code = NULL,
vendor_address_country_code = NULL,
vendor_address_country_name = NULL,
vendor_address_state_code = NULL,
vendor_address_state_name = NULL,
vendor_address_zip_code = NULL,
number_of_offers_received = NULL,
extent_competed = NULL,
sort_item = "Signed Date",
sort_descending = T,
return_message = T,
...) {
base_url <-
"https://www.fpds.gov/ezsearch/FEEDS/ATOM?FEEDNAME=PUBLIC&q="
df_parameters <-
.fpds_api_call_params(
contract_id = contract_id,
global_vendor_name = global_vendor_name,
award_type = award_type,
contracting_agency_id = contracting_agency_id,
contracting_agency_name = contracting_agency_name,
contracting_office_id = contracting_office_id,
contract_type_description = contract_type_description,
contracting_office_name = contracting_office_name,
funding_agency_id = funding_agency_id,
funding_office_id = funding_office_id,
funding_office_name = funding_office_name,
subcontract_plan = subcontract_plan,
agency_code = agency_code,
agency_name = agency_name,
department_id = department_id,
department_name = department_name,
last_mod_date = last_mod_date,
last_modified_by = last_modified_by,
award_completion_date = award_completion_date,
created_date = created_date,
signed_date = signed_date,
effective_date = effective_date,
estimated_completion_date = estimated_completion_date,
number_of_offers_received = number_of_offers_received,
base_exercised_options_value = base_exercised_options_value,
current_contract_value = current_contract_value,
dollars_obligated = dollars_obligated,
contract_value = contract_value,
fee_range_lower_value = fee_range_lower_value,
fee_range_upper_value = fee_range_upper_value,
fixed_fee_value = fixed_fee_value,
obligated_amount = obligated_amount,
total_current_contract_value = total_current_contract_value,
total_dollars_obligated = total_dollars_obligated,
total_non_government_value = total_non_government_value,
total_ultimate_contract_value = total_ultimate_contract_value,
ultimate_contract_value = ultimate_contract_value,
contract_fiscal_year = contract_fiscal_year,
type_of_contract_pricing = type_of_contract_pricing,
award_status = award_status,
contract_type = contract_type,
description_of_requirement = description_of_requirement,
reason_for_modification = reason_for_modification,
legislative_mandates = legislative_mandates,
local_area_set_aside = local_area_set_aside,
socio_economic_indicators = socio_economic_indicators,
multiyear_contract = multiyear_contract,
national_interest_code = national_interest_code,
national_interest_description = national_interest_description,
principal_naics_code = principal_naics_code,
naics_description = naics_description,
product_or_service_code = product_or_service_code,
product_or_service_description = product_or_service_description,
performance_district_code = performance_district_code,
performance_country = performance_country,
pop_state_name = performance_state_name,
vendor_address_city = vendor_address_city,
vendor_congress_district_code = vendor_congress_district_code,
vendor_address_country_code = vendor_address_country_code,
vendor_address_country_name = vendor_address_country_name,
vendor_duns_number = vendor_duns_number,
vendor_doing_business_as_name = vendor_doing_business_as_name,
vendor_name = vendor_name,
vendor_address_state_code = vendor_address_state_code,
vendor_address_state_name = vendor_address_state_name,
vendor_address_zip_code = vendor_address_zip_code,
ref_idv_contract_id = ref_idv_contract_id,
ref_idv_agency_id = ref_idv_agency_id,
modification_number = modification_number,
created_by = created_by,
research = research,
extent_competed = extent_competed,
performance_state_name = performance_state_name,
cancellation_date = cancellation_date,
destroy_date = destroy_date,
final_invoice_paid_date = final_invoice_paid_date,
funded_through_date = funded_through_date,
last_modified_date = last_modified_date,
physical_completion_date = physical_completion_date,
reveal_date = reveal_date,
solicitation_issue_date = solicitation_issue_date,
sys_last_modified_date = sys_last_modified_date,
vendor_registration_date = vendor_registration_date,
vendor_renewal_date = vendor_renewal_date,
solicitation_procedure = solicitation_procedure,
return_query = F,
...
) %>%
mutate_all(str_to_upper)
sort_slug <-
.sort_slug(sort_method = sort_item, sort_descending = sort_descending)
df_values <-
df_parameters %>%
gather(param, value)
query <-
df_values %>%
mutate(
value = value %>% str_to_upper() %>% map_chr(URLencode),
param = str_to_upper(param)
) %>%
unite(slug, param, value, sep = ":") %>%
pull(slug) %>%
str_c(collapse = "%20") %>%
map_chr(URLencode)
url <-
glue::glue("{base_url}{query}&{sort_slug}") %>% as.character()
data <-
df_parameters %>%
.munge_fpds_names()
names(data) <-
names(data) %>% str_c("Search")
data <-
data %>%
mutate(urlBase = url)
df_urls <-
.generate_atom_urls(url = url) %>%
mutate(urlBase = url)
df_urls %>%
left_join(data, by = "urlBase") %>%
select(-urlBase)
}
.generate_fpds_advanced_queries <-
function(vendor_doing_business_as_name = NA,
award_type = NA,
global_vendor_name = NA,
vendor_name = NA,
parent_vendor_name = NA,
vendor_duns_number = NA,
parent_duns_number = NA,
agency_name = NA,
contracting_office_name = NA,
contracting_agency_name = NA,
principal_naics_code = NA,
award_status = NA,
solicitation_procedure = NA,
contract_type = NA,
contract_type_description = NA,
type_of_contract_pricing = NA,
contract_id = NA,
subcontract_plan = NA,
ref_idv_contract_id = NA,
ref_idv_agency_id = NA,
contracting_agency_id = NA,
contracting_office_id = NA,
funding_agency_id = NA,
funding_office_id = NA,
funding_office_name = NA,
agency_code = NA,
department_id = NA,
department_name = NA,
research = NA,
last_mod_date = NA,
last_modified_by = NA,
award_completion_date = NA,
created_date = NA,
signed_date = NA,
effective_date = NA,
estimated_completion_date = NA,
cancellation_date = NA,
destroy_date = NA,
final_invoice_paid_date = NA,
funded_through_date = NA,
last_modified_date = NA,
physical_completion_date = NA,
reveal_date = NA,
solicitation_issue_date = NA,
sys_last_modified_date = NA,
vendor_registration_date = NA,
vendor_renewal_date = NA,
base_exercised_options_value = NA,
current_contract_value = NA,
dollars_obligated = NA,
contract_value = NA,
fee_range_lower_value = NA,
fee_range_upper_value = NA,
fixed_fee_value = NA,
obligated_amount = NA,
total_current_contract_value = NA,
total_dollars_obligated = NA,
total_non_government_value = NA,
total_ultimate_contract_value = NA,
ultimate_contract_value = NA,
contract_fiscal_year = NA,
created_by = NA,
description_of_requirement = NA,
reason_for_modification = NA,
legislative_mandates = NA,
local_area_set_aside = NA,
socio_economic_indicators = NA,
multiyear_contract = NA,
national_interest_code = NA,
product_or_service_code = NA,
performance_district_code = NA,
performance_country = NA,
performance_state_name = NA,
vendor_address_city = NA,
vendor_congress_district_code = NA,
vendor_address_country_code = NA,
vendor_address_country_name = NA,
vendor_address_state_code = NA,
vendor_address_state_name = NA,
vendor_address_zip_code = NA,
extent_competed = NA,
number_of_offers_received = NA,
sort_item = "Signed Date",
sort_descending = T,
return_message = T,
...) {
options(warn = -1)
if (!is.na(contract_id) %>% sum(na.rm = T) > 0) {
contract_id <-
contract_id %>% str_remove_all("\\-")
}
if (!is.na(last_mod_date) %>% sum(na.rm = T) > 0 &
length(last_mod_date) == 2) {
last_mod_date <- .shorten_dates(dates = last_mod_date)
}
if (!is.na(award_completion_date) %>% sum(na.rm = T) > 0 &
length(award_completion_date) == 2) {
award_completion_date <-
.shorten_dates(dates = award_completion_date)
}
if (!is.na(created_date) %>% sum(na.rm = T) > 0 &
length(created_date) == 2) {
created_date <- .shorten_dates(dates = created_date)
}
if (!is.na(signed_date) %>% sum(na.rm = T) > 0 &
length(signed_date) == 2) {
signed_date <- .shorten_dates(dates = signed_date)
}
if (!is.na(effective_date) %>% sum(na.rm = T) > 0 &
length(effective_date) == 2) {
effective_date <- .shorten_dates(dates = effective_date)
}
if (!is.na(estimated_completion_date) %>% sum(na.rm = T) > 0 &
length(estimated_completion_date) == 2) {
estimated_completion_date <-
.shorten_dates(dates = estimated_completion_date)
}
if (!is.na(cancellation_date) %>% sum(na.rm = T) > 0 &
length(cancellation_date) == 2) {
cancellation_date <- .shorten_dates(dates = cancellation_date)
}
if (!is.na(destroy_date) %>% sum(na.rm = T) > 0 &
length(destroy_date) == 2) {
destroy_date <- .shorten_dates(dates = destroy_date)
}
if (!is.na(final_invoice_paid_date) %>% sum(na.rm = T) > 0 &
length(final_invoice_paid_date) == 2) {
final_invoice_paid_date <-
.shorten_dates(dates = final_invoice_paid_date)
}
if (!is.na(funded_through_date) %>% sum(na.rm = T) > 0 &
length(funded_through_date) == 2) {
funded_through_date <- .shorten_dates(dates = funded_through_date)
}
if (!is.na(last_modified_date) %>% sum(na.rm = T) > 0 &
length(last_modified_date) == 2) {
last_modified_date <- .shorten_dates(dates = last_modified_date)
}
if (!is.na(physical_completion_date) %>% sum(na.rm = T) > 0 &
length(physical_completion_date) == 2) {
physical_completion_date <-
.shorten_dates(dates = physical_completion_date)
}
if (!is.na(vendor_duns_number)) {
vendor_duns_number <-
.pad_duns(duns = vendor_duns_number, zero_base = 9) %>% pull(slugDUNS)
}
if (!is.na(parent_duns_number)) {
parent_duns_number <-
.pad_duns(duns = parent_duns_number, zero_base = 9) %>% pull(slugDUNS)
}
if (!is.na(reveal_date) %>% sum(na.rm = T) > 0 &
length(reveal_date) == 2) {
reveal_date <- .shorten_dates(dates = reveal_date)
}
if (!is.na(solicitation_issue_date) %>% sum(na.rm = T) > 0 &
length(solicitation_issue_date) == 2) {
solicitation_issue_date <-
.shorten_dates(dates = solicitation_issue_date)
}
if (!is.na(sys_last_modified_date) %>% sum(na.rm = T) > 0 &
length(sys_last_modified_date) == 2) {
sys_last_modified_date <-
.shorten_dates(dates = sys_last_modified_date)
}
if (!is.na(vendor_registration_date) %>% sum(na.rm = T) > 0 &
length(vendor_registration_date) == 2) {
vendor_registration_date <-
.shorten_dates(dates = vendor_registration_date)
}
if (!is.na(vendor_renewal_date) %>% sum(na.rm = T) > 0 &
length(signed_date) == 2) {
vendor_renewal_date <-
.shorten_dates(dates = vendor_renewal_date)
}
if (!is.na(base_exercised_options_value) %>% sum(na.rm = T) > 0 &
length(base_exercised_options_value) == 2) {
base_exercised_options_value <-
.shorten_amounts(base_exercised_options_value)
}
if (!is.na(current_contract_value) %>% sum(na.rm = T) > 0 &
length(current_contract_value) == 2) {
current_contract_value <- .shorten_amounts(current_contract_value)
}
if (!is.na(dollars_obligated) %>% sum(na.rm = T) > 0 &
length(dollars_obligated) == 2) {
dollars_obligated <- .shorten_amounts(dollars_obligated)
}
if (!is.na(contract_value) %>% sum(na.rm = T) > 0 &
length(contract_value) == 2) {
contract_value <- .shorten_amounts(contract_value)
}
if (!is.na(fee_range_lower_value) %>% sum(na.rm = T) > 0 &
length(fee_range_lower_value) == 2) {
fee_range_lower_value <- .shorten_amounts(fee_range_lower_value)
}
if (!is.na(fee_range_upper_value) %>% sum(na.rm = T) > 0 &
length(fee_range_upper_value) == 2) {
fee_range_upper_value <- .shorten_amounts(fee_range_upper_value)
}
if (!is.na(fixed_fee_value) %>% sum(na.rm = T) > 0 &
length(fixed_fee_value) == 2) {
fixed_fee_value <- .shorten_amounts(fixed_fee_value)
}
if (!is.na(obligated_amount) %>% sum(na.rm = T) > 0 &
length(obligated_amount) == 2) {
obligated_amount <- .shorten_amounts(obligated_amount)
}
if (!is.na(total_current_contract_value) %>% sum(na.rm = T) > 0 &
length(total_current_contract_value) == 2) {
total_current_contract_value <-
.shorten_amounts(total_current_contract_value)
}
if (!is.na(total_dollars_obligated) %>% sum(na.rm = T) > 0 &
length(total_dollars_obligated) == 2) {
total_dollars_obligated <- .shorten_amounts(total_dollars_obligated)
}
if (!is.na(total_non_government_value) %>% sum(na.rm = T) > 0 &
length(total_non_government_value) == 2) {
total_non_government_value <-
.shorten_amounts(total_non_government_value)
}
if (!is.na(total_ultimate_contract_value) %>% sum(na.rm = T) > 0 &
length(total_ultimate_contract_value) == 2) {
total_ultimate_contract_value <-
.shorten_amounts(total_ultimate_contract_value)
}
if (!is.na(ultimate_contract_value) %>% sum(na.rm = T) > 0 &
length(ultimate_contract_value) == 2) {
ultimate_contract_value <-
.shorten_amounts(ultimate_contract_value)
}
df_inputs <-
expand.grid(
vendor_doing_business_as_name = vendor_doing_business_as_name,
global_vendor_name = global_vendor_name,
vendor_name = vendor_name,
vendor_duns_number = vendor_duns_number,
agency_name = agency_name,
award_type = award_type,
contracting_office_name = contracting_office_name,
contracting_agency_name = contracting_agency_name,
principal_naics_code = principal_naics_code,
parent_duns_number = parent_duns_number,
parent_vendor_name = parent_vendor_name,
award_status = award_status,
contract_type = contract_type,
contract_type_description = contract_type_description,
type_of_contract_pricing = type_of_contract_pricing,
contract_id = contract_id,
ref_idv_contract_id = ref_idv_contract_id,
ref_idv_agency_id = ref_idv_agency_id ,
contracting_agency_id = contracting_agency_id,
contracting_office_id = contracting_office_id,
funding_agency_id = funding_agency_id,
funding_office_id = funding_office_id,
subcontract_plan = subcontract_plan,
funding_office_name = funding_office_name ,
agency_code = agency_code,
department_id = department_id,
department_name = department_name,
last_mod_date = last_mod_date,
last_modified_by = last_modified_by,
award_completion_date = award_completion_date,
created_date = created_date,
signed_date = signed_date ,
effective_date = effective_date,
estimated_completion_date = estimated_completion_date,
cancellation_date = cancellation_date,
destroy_date = destroy_date,
final_invoice_paid_date = final_invoice_paid_date,
funded_through_date = funded_through_date,
last_modified_date = last_modified_date,
physical_completion_date = physical_completion_date,
reveal_date = reveal_date,
solicitation_issue_date = solicitation_issue_date,
sys_last_modified_date = sys_last_modified_date,
vendor_registration_date = vendor_registration_date,
vendor_renewal_date = vendor_renewal_date,
base_exercised_options_value = base_exercised_options_value,
current_contract_value = current_contract_value,
dollars_obligated = dollars_obligated,
contract_value = contract_value,
fee_range_lower_value = fee_range_lower_value,
fee_range_upper_value = fee_range_upper_value,
fixed_fee_value = fixed_fee_value,
obligated_amount = obligated_amount,
total_current_contract_value = total_current_contract_value,
total_dollars_obligated = total_dollars_obligated,
total_non_government_value = total_non_government_value,
total_ultimate_contract_value = total_ultimate_contract_value,
ultimate_contract_value = ultimate_contract_value,
contract_fiscal_year = contract_fiscal_year,
created_by = created_by,
description_of_requirement = description_of_requirement,
reason_for_modification = reason_for_modification ,
legislative_mandates = legislative_mandates,
local_area_set_aside = local_area_set_aside ,
socio_economic_indicators = socio_economic_indicators ,
multiyear_contract = multiyear_contract,
national_interest_code = national_interest_code,
product_or_service_code = product_or_service_code,
performance_district_code = performance_district_code,
performance_country = performance_country ,
performance_state_name = performance_state_name,
vendor_address_city = vendor_address_city,
vendor_congress_district_code = vendor_congress_district_code,
vendor_address_country_code = vendor_address_country_code,
vendor_address_country_name = vendor_address_country_name,
vendor_address_state_code = vendor_address_state_code,
vendor_address_state_name = vendor_address_state_name,
vendor_address_zip_code = vendor_address_zip_code,
research = research,
solicitation_procedure = solicitation_procedure,
extent_competed = extent_competed,
number_of_offers_received = number_of_offers_received,
stringsAsFactors = F,
...
) %>%
as_tibble()
if (df_inputs %>% gather(item, value) %>% filter(!is.na(value)) %>% nrow() == 0) {
stop("Please enter FPDS search parameter")
}
data <-
1:nrow(df_inputs) %>%
map_dfr(function(x) {
df_row <-
df_inputs %>% dplyr::slice(x) %>% as_tibble() %>% mutate_if(is.factor, as.character)
if (is.na(df_row$vendor_doing_business_as_name)) {
vendor_doing_business_as_name_slug <- NULL
} else {
vendor_doing_business_as_name_slug <-
df_row$vendor_doing_business_as_name
}
if (is.na(df_row$global_vendor_name)) {
global_vendor_name_slug <- NULL
} else {
global_vendor_name_slug <- df_row$global_vendor_name
}
if (is.na(df_row$subcontract_plan)) {
subcontract_plan_slug <- NULL
} else {
subcontract_plan_slug <- df_row$subcontract_plan
}
if (is.na(df_row$solicitation_procedure)) {
solicitation_procedure_slug <- NULL
} else {
solicitation_procedure_slug <- df_row$solicitation_procedure
}
if (is.na(df_row$research)) {
research_slug <- NULL
} else {
research_slug <- df_row$research
}
if (is.na(df_row$award_type)) {
award_type_slug <- NULL
} else {
award_type_slug <- df_row$award_type
}
if (is.na(df_row$vendor_name)) {
vendor_name_slug <- NULL
} else {
vendor_name_slug <- df_row$vendor_name
}
if (is.na(df_row$parent_vendor_name)) {
parent_vendor_name_slug <- NULL
} else {
parent_vendor_name_slug <- df_row$parent_vendor_name
}
if (is.na(df_row$vendor_duns_number)) {
vendor_duns_number_slug <- NULL
} else {
vendor_duns_number_slug <- df_row$vendor_duns_number
}
if (is.na(df_row$parent_duns_number)) {
parent_duns_number_slug <- NULL
} else {
parent_duns_number_slug <- df_row$parent_duns_number
}
if (is.na(df_row$agency_name)) {
agency_name_slug <- NULL
} else {
agency_name_slug <- df_row$agency_name
}
if (is.na(df_row$contracting_office_name)) {
contracting_office_name_slug <- NULL
} else {
contracting_office_name_slug <- df_row$contracting_office_name
}
if (is.na(df_row$contracting_agency_name)) {
contracting_agency_name_slug <- NULL
} else {
contracting_agency_name_slug <- df_row$contracting_agency_name
}
if (is.na(df_row$principal_naics_code)) {
principal_naics_code_slug <- NULL
} else {
principal_naics_code_slug <- df_row$principal_naics_code
}
if (is.na(df_row$award_status)) {
award_status_slug <- NULL
} else {
award_status_slug <- df_row$award_status
}
if (is.na(df_row$contract_type)) {
contract_type_slug <- NULL
} else {
contract_type_slug <- df_row$contract_type
}
if (is.na(df_row$contract_type_description)) {
contract_type_description_slug <- NULL
} else {
contract_type_description_slug <- df_row$contract_type_description
}
if (is.na(df_row$type_of_contract_pricing)) {
type_of_contract_pricing_slug <- NULL
} else {
type_of_contract_pricing_slug <- df_row$type_of_contract_pricing
}
if (is.na(df_row$contract_id)) {
contract_id_slug <- NULL
} else {
contract_id_slug <- df_row$contract_id
}
if (is.na(df_row$ref_idv_contract_id)) {
ref_idv_contract_id_slug <- NULL
} else {
ref_idv_contract_id_slug <- df_row$ref_idv_contract_id
}
if (is.na(df_row$ref_idv_agency_id)) {
ref_idv_agency_id_slug <- NULL
} else {
ref_idv_agency_id_slug <- df_row$ref_idv_agency_id
}
if (is.na(df_row$contracting_agency_id)) {
contracting_agency_id_slug <- NULL
} else {
contracting_agency_id_slug <- df_row$contracting_agency_id
}
if (is.na(df_row$contracting_office_id)) {
contracting_office_id_slug <- NULL
} else {
contracting_office_id_slug <- df_row$contracting_office_id
}
if (is.na(df_row$funding_agency_id)) {
funding_agency_id_slug <- NULL
} else {
funding_agency_id_slug <- df_row$funding_agency_id
}
if (is.na(df_row$funding_office_id)) {
funding_office_id_slug <- NULL
} else {
funding_office_id_slug <- df_row$funding_office_id
}
if (is.na(df_row$funding_office_name)) {
funding_office_name_slug <- NULL
} else {
funding_office_name_slug <- df_row$funding_office_name
}
if (is.na(df_row$agency_code)) {
agency_code_slug <- NULL
} else {
agency_code_slug <- df_row$agency_code
}
if (is.na(df_row$department_id)) {
department_id_slug <- NULL
} else {
department_id_slug <- df_row$department_id
}
if (is.na(df_row$department_name)) {
department_name_slug <- NULL
} else {
department_name_slug <- df_row$department_name
}
if (is.na(df_row$last_mod_date)) {
last_mod_date_slug <- NULL
} else {
last_mod_date_slug <- df_row$last_mod_date
}
if (is.na(df_row$last_modified_by)) {
last_modified_by_slug <- NULL
} else {
last_modified_by_slug <- df_row$last_modified_by
}
if (is.na(df_row$award_completion_date)) {
award_completion_date_slug <- NULL
} else {
award_completion_date_slug <- df_row$award_completion_date
}
if (is.na(df_row$created_date)) {
created_date_slug <- NULL
} else {
created_date_slug <- df_row$created_date
}
if (is.na(df_row$signed_date)) {
signed_date_slug <- NULL
} else {
signed_date_slug <- df_row$signed_date
}
if (is.na(df_row$effective_date)) {
effective_date_slug <- NULL
} else {
effective_date_slug <- df_row$effective_date
}
if (is.na(df_row$estimated_completion_date)) {
estimated_completion_date_slug <- NULL
} else {
estimated_completion_date_slug <- df_row$estimated_completion_date
}
if (is.na(df_row$cancellation_date)) {
cancellation_date_slug <- NULL
} else {
cancellation_date_slug <- df_row$cancellation_date
}
if (is.na(df_row$destroy_date)) {
destroy_date_slug <- NULL
} else {
destroy_date_slug <- df_row$destroy_date
}
if (is.na(df_row$final_invoice_paid_date)) {
final_invoice_paid_date_slug <- NULL
} else {
final_invoice_paid_date_slug <- df_row$final_invoice_paid_date
}
if (is.na(df_row$funded_through_date)) {
funded_through_date_slug <- NULL
} else {
funded_through_date_slug <- df_row$funded_through_date
}
if (is.na(df_row$last_modified_date)) {
last_modified_date_slug <- NULL
} else {
last_modified_date_slug <- df_row$last_modified_date
}
if (is.na(df_row$physical_completion_date)) {
physical_completion_date_slug <- NULL
} else {
physical_completion_date_slug <- df_row$physical_completion_date
}
if (is.na(df_row$reveal_date)) {
reveal_date_slug <- NULL
} else {
reveal_date_slug <- df_row$reveal_date
}
if (is.na(df_row$solicitation_issue_date)) {
solicitation_issue_date_slug <- NULL
} else {
solicitation_issue_date_slug <- df_row$solicitation_issue_date
}
if (is.na(df_row$sys_last_modified_date)) {
sys_last_modified_date_slug <- NULL
} else {
sys_last_modified_date_slug <- df_row$sys_last_modified_date
}
if (is.na(df_row$vendor_registration_date)) {
vendor_registration_date_slug <- NULL
} else {
vendor_registration_date_slug <- df_row$vendor_registration_date
}
if (is.na(df_row$vendor_renewal_date)) {
vendor_renewal_date_slug <- NULL
} else {
vendor_renewal_date_slug <-
df_row$vendor_revendor_renewal_dategistration_date
}
if (is.na(df_row$obligated_amount)) {
obligated_amount_slug <- NULL
} else {
obligated_amount_slug <- df_row$obligated_amount
}
if (is.na(df_row$base_exercised_options_value)) {
base_exercised_options_value_slug <- NULL
} else {
base_exercised_options_value_slug <-
df_row$base_exercised_options_value
}
if (is.na(df_row$ultimate_contract_value)) {
ultimate_contract_value_slug <- NULL
} else {
ultimate_contract_value_slug <- df_row$ultimate_contract_value
}
if (is.na(df_row$created_by)) {
created_by_slug <- NULL
} else {
created_by_slug <- df_row$created_by
}
if (is.na(df_row$description_of_requirement)) {
description_of_requirement_slug <- NULL
} else {
description_of_requirement_slug <- df_row$description_of_requirement
}
if (is.na(df_row$legislative_mandates)) {
legislative_mandates_slug <- NULL
} else {
legislative_mandates_slug <- df_row$legislative_mandates
}
if (is.na(df_row$local_area_set_aside)) {
local_area_set_aside_slug <- NULL
} else {
local_area_set_aside_slug <- df_row$local_area_set_aside
}
if (is.na(df_row$socio_economic_indicators)) {
socio_economic_indicators_slug <- NULL
} else {
socio_economic_indicators_slug <- df_row$socio_economic_indicators
}
if (is.na(df_row$multiyear_contract)) {
multiyear_contract_slug <- NULL
} else {
multiyear_contract_slug <- df_row$multiyear_contract
}
if (is.na(df_row$national_interest_code)) {
national_interest_code_slug <- NULL
} else {
national_interest_code_slug <- df_row$national_interest_code
}
if (is.na(df_row$product_or_service_code)) {
product_or_service_code_slug <- NULL
} else {
product_or_service_code_slug <- df_row$product_or_service_code
}
if (is.na(df_row$performance_district_code)) {
performance_district_code_slug <- NULL
} else {
performance_district_code_slug <- df_row$performance_district_code
}
if (is.na(df_row$performance_country)) {
performance_country_slug <- NULL
} else {
performance_country_slug <- df_row$performance_country
}
if (is.na(df_row$performance_state_name)) {
performance_state_name_slug <- NULL
} else {
performance_state_name_slug <- df_row$performance_state_name
}
if (is.na(df_row$vendor_address_city)) {
vendor_address_city_slug <- NULL
} else {
vendor_address_city_slug <- df_row$vendor_address_city
}
if (is.na(df_row$vendor_congress_district_code)) {
vendor_congress_district_code_slug <- NULL
} else {
vendor_congress_district_code_slug <-
df_row$vendor_congress_district_code
}
if (is.na(df_row$vendor_address_country_code)) {
vendor_address_country_code_slug <- NULL
} else {
vendor_address_country_code_slug <-
df_row$vendor_address_country_code
}
if (is.na(vendor_address_country_name)) {
vendor_address_country_name_slug <- NULL
} else {
vendor_address_country_name_slug <-
df_row$vendor_address_country_name
}
if (is.na(df_row$vendor_address_state_code)) {
vendor_address_state_code_slug <- NULL
} else {
vendor_address_state_code_slug <- df_row$vendor_address_state_code
}
if (is.na(df_row$vendor_address_state_name)) {
vendor_address_state_name_slug <- NULL
} else {
vendor_address_state_name_slug <- df_row$vendor_address_state_name
}
if (is.na(df_row$vendor_address_zip_code)) {
vendor_address_zip_code_slug <- NULL
} else {
vendor_address_zip_code_slug <- df_row$vendor_address_zip_code
}
if (is.na(df_row$reason_for_modification)) {
reason_for_modification_slug <- NULL
} else {
reason_for_modification_slug <- df_row$reason_for_modification
}
if (is.na(current_contract_value)) {
current_contract_value_slug <- NULL
} else {
current_contract_value_slug <- df_row$current_contract_value
}
if (is.na(dollars_obligated)) {
dollars_obligated_slug <- NULL
} else {
dollars_obligated_slug <- df_row$dollars_obligated
}
if (is.na(contract_value)) {
contract_value_slug <- NULL
} else {
contract_value_slug <- df_row$contract_value
}
if (is.na(fee_range_lower_value)) {
fee_range_lower_value_slug <- NULL
} else {
fee_range_lower_value_slug <- df_row$fee_range_lower_value
}
if (is.na(fee_range_upper_value)) {
fee_range_upper_value_slug <- NULL
} else {
}
if (is.na(fixed_fee_value)) {
fixed_fee_value_slug <- NULL
} else {
fixed_fee_value_slug <- df_row$fixed_fee_value
}
if (is.na(total_current_contract_value)) {
total_current_contract_value_slug <- NULL
} else {
total_current_contract_value_slug <-
df_row$total_current_contract_value
}
if (is.na(total_dollars_obligated)) {
total_dollars_obligated_slug <- NULL
} else {
total_dollars_obligated_slug <- df_row$total_dollars_obligated
}
if (is.na(total_non_government_value)) {
total_non_government_value_slug <- NULL
} else {
total_non_government_value_slug <- df_row$total_non_government_value
}
if (is.na(total_ultimate_contract_value)) {
total_ultimate_contract_value_slug <- NULL
} else {
total_ultimate_contract_value_slug <-
df_row$total_ultimate_contract_value
}
if (is.na(contract_fiscal_year)) {
contract_fiscal_year_slug <- NULL
} else {
contract_fiscal_year_slug <- df_row$contract_fiscal_year
}
if (is.na(extent_competed)) {
extent_competed_slug <- NULL
} else {
extent_competed_slug <- df_row$extent_competed
}
if (is.na(number_of_offers_received)) {
number_of_offers_received_slug <- NULL
} else {
number_of_offers_received_slug <- df_row$number_of_offers_received
}
.generate_fpds_advanced_query(
vendor_doing_business_as_name = vendor_doing_business_as_name_slug,
global_vendor_name = global_vendor_name_slug,
vendor_name = vendor_name_slug,
parent_vendor_name = parent_vendor_name_slug,
vendor_duns_number = vendor_duns_number_slug,
parent_duns_number = parent_duns_number_slug,
agency_name = agency_name_slug,
contracting_office_name = contracting_office_name_slug,
contracting_agency_name = contracting_agency_name_slug,
principal_naics_code = principal_naics_code_slug,
award_status = award_status_slug,
contract_type = contract_type_slug,
contract_type_description = contract_type_description_slug,
type_of_contract_pricing = type_of_contract_pricing_slug,
contract_id = contract_id_slug,
ref_idv_contract_id = ref_idv_contract_id_slug,
ref_idv_agency_id = ref_idv_agency_id_slug,
contracting_agency_id = contracting_agency_id_slug,
contracting_office_id = contracting_office_id_slug,
funding_agency_id = funding_agency_id_slug,
funding_office_id = funding_office_id_slug,
funding_office_name = funding_office_name_slug,
agency_code = agency_code_slug,
department_id = department_id_slug,
department_name = department_name_slug,
research = research_slug,
last_mod_date = last_mod_date_slug,
last_modified_by = last_modified_by_slug,
award_completion_date = award_completion_date_slug,
created_date = created_date_slug,
signed_date = signed_date_slug,
effective_date = effective_date_slug,
estimated_completion_date = estimated_completion_date_slug,
obligated_amount = obligated_amount_slug,
base_exercised_options_value = base_exercised_options_value_slug,
ultimate_contract_value = ultimate_contract_value_slug,
created_by = created_by_slug,
description_of_requirement = description_of_requirement_slug,
reason_for_modification = reason_for_modification_slug,
legislative_mandates = legislative_mandates_slug,
local_area_set_aside = local_area_set_aside_slug,
socio_economic_indicators = socio_economic_indicators_slug,
multiyear_contract = multiyear_contract_slug,
national_interest_code = national_interest_code_slug,
product_or_service_code = product_or_service_code_slug,
performance_district_code = performance_district_code_slug,
performance_country = performance_country_slug,
performance_state_name = performance_state_name_slug,
vendor_address_city = vendor_address_city_slug,
vendor_congress_district_code = vendor_congress_district_code_slug,
vendor_address_country_code = vendor_address_country_code_slug,
vendor_address_country_name = vendor_address_country_name_slug,
vendor_address_state_code = vendor_address_state_code_slug,
vendor_address_state_name = vendor_address_state_name_slug,
vendor_address_zip_code = vendor_address_zip_code_slug,
sort_item = sort_item,
sort_descending = sort_descending,
return_message = return_message,
cancellation_date = cancellation_date_slug,
destroy_date = destroy_date_slug,
final_invoice_paid_date = final_invoice_paid_date_slug,
funded_through_date = funded_through_date_slug,
last_modified_date = last_modified_date_slug,
physical_completion_date = physical_completion_date_slug,
reveal_date = reveal_date_slug,
solicitation_issue_date = solicitation_issue_date_slug,
sys_last_modified_date = sys_last_modified_date_slug,
vendor_registration_date = vendor_registration_date_slug,
vendor_renewal_date = vendor_renewal_date_slug,
award_type = award_type_slug,
solicitation_procedure = solicitation_procedure_slug,
subcontract_plan = subcontract_plan_slug,
current_contract_value = current_contract_value_slug,
dollars_obligated = dollars_obligated_slug,
contract_value = contract_value_slug,
fee_range_lower_value = fee_range_lower_value_slug,
fee_range_upper_value = fee_range_upper_value_slug,
fixed_fee_value = fixed_fee_value_slug,
total_current_contract_value = total_current_contract_value_slug,
total_dollars_obligated = total_dollars_obligated_slug,
total_non_government_value = total_non_government_value_slug,
total_ultimate_contract_value = total_ultimate_contract_value_slug,
contract_fiscal_year = contract_fiscal_year_slug,
extent_competed = extent_competed_slug,
number_of_offers_received = number_of_offers_received_slug,
...
)
}) %>%
dplyr::select(which(colMeans(is.na(.)) < 1))
data
}
.count_final_atom_page <-
function(url = "https://www.fpds.gov/ezsearch/FEEDS/ATOM?FEEDNAME=PUBLIC&q=%22BRESLER%22&start=10&sortBy=SIGNED_DATE&start=10&sortBy=SIGNED_DATE") {
doc <- read_xml(url)
xml_nodes <-
doc %>%
xml_contents()
tables <-
xml_nodes %>%
xml_name()
df_table_nodes <-
tibble(table = tables) %>%
mutate(idTable = 1:n())
df_ids <-
df_table_nodes %>%
filter(table == "entry")
nrow(df_ids)
}
.generate_fpds_ft_query <-
function(term = "BRESLER",
use_quote = T,
return_message = T) {
base_url <-
"https://www.fpds.gov/ezsearch/FEEDS/ATOM?FEEDNAME=PUBLIC&q="
if (use_quote) {
slug <-
glue::glue('"{term}"') %>% as.character()
} else {
slug <- term
}
query <-
URLencode(slug)
url <- glue::glue("{base_url}{query}") %>% as.character()
df_urls <-
.generate_atom_urls(url = url) %>%
mutate(urlBase = url)
final_url <- df_urls %>%
dplyr::slice(nrow(df_urls)) %>%
pull(urlFPDSAtom)
count_final <- .count_final_atom_page(url = final_url)
total_results <-
(10 * (length(df_urls$urlFPDSAtom) - 1)) + count_final
if (total_results == 0) {
glue("No results for {term}") %>% message()
data <- tibble(
slugSearch = slug,
termSearch = term,
isQuoted = use_quote,
countResults = total_results
)
return()
}
if (return_message) {
slug <- URLdecode(query)
glue("FPDS search of {slug} has {comma(total_results, digits = 0)} results") %>% message()
}
df_urls %>%
mutate(
slugSearch = slug,
termSearch = term,
isQuoted = use_quote,
countResults = total_results
) %>%
select(termSearch, slugSearch, isQuoted, countResults, everything()) %>%
select(-one_of("idSequence", "urlBase"))
}
# atom query -------------------------------------------------------------------
#' FDPS Free Text Search
#'
#' @param terms
#' @param use_quote
#' @param parse_contracts
#' @param return_message
#' @param clean_entity_column
#' @param snake_names
#' @param keep_key_columns
#' @param exclude_bloat
#' @param unformat
#'
#' @return
#' @export
#'
#' @examples
#' fpds_free_text_search(terms = "Palantir Technologies")
fpds_free_text_search <-
function(terms = NULL,
use_quote = T,
use_future = F,
show_progress = T,
parse_contracts = F,
keep_key_columns = F,
clean_entity_column = T,
exclude_bloat = F,
snake_names = F,
clean_address = T,
unformat = F,
return_message = T) {
if (length(terms) == 0) {
stop("Enter search terms")
}
df_terms <-
expand.grid(term = terms,
use_quote = use_quote,
stringsAsFactors = F) %>%
as_tibble()
df_urls <-
1:nrow(df_terms) %>%
map_dfr(function(x) {
df_row <- df_terms %>% dplyr::slice(x)
term <- df_row$term %>% str_to_upper()
quote <- df_row$use_quote
d <- .generate_fpds_ft_query(term = term,
use_quote = quote)
d
})
if (!parse_contracts) {
return(df_urls)
}
urls <- df_urls$urlFPDSAtom
all_data <-
parse_fpds_atom_urls(
urls = urls,
return_message = return_message,
keep_key_columns = keep_key_columns,
exclude_bloat = exclude_bloat,
clean_entity_column = clean_entity_column,
unformat = unformat,
use_future = use_future,
clean_address = clean_address,
snake_names = snake_names
)
all_data$urlFPDSAtom %>% unique()
all_data <-
all_data %>%
mutate(id = urlFPDSAtom %>% str_to_upper()) %>%
left_join(
df_urls %>%
mutate(id = urlFPDSAtom %>% str_to_upper()) %>%
select(termSearch, countResults, countResults, id)
,
by = "id"
) %>%
select(termSearch, countResults, everything()) %>%
select(-id)
group_names <- c(
"idContract",
"nameContract",
"nameAgencyAward",
"dateContractSigned",
"amountObligation",
"numberTransaction",
"idContractIDV",
"nameVendorLegal",
"idDUNSParent"
)
group_cols <- names(all_data)[names(all_data) %in% group_names]
df_counts <- all_data %>%
group_by(!!!syms(group_cols)) %>%
summarise(
termsSearch = unique(termSearch) %>% sort() %>% str_c(collapse = " | "),
countMatchTerms = length(unique(termSearch))
)
all_data <-
all_data %>%
select(-one_of(c(
"termSearch", "countResults", "slugSearch", "isQuoted"
))) %>%
distinct() %>%
left_join(df_counts, by = group_cols) %>%
select(termsSearch, countMatchTerms, everything())
if (snake_names) {
all_data <- all_data %>%
janitor::clean_names()
}
all_data
}
#' FDPS Advanced Search
#'
#' @param vendor_doing_business_as_name
#' @param global_vendor_name
#' @param vendor_name
#' @param vendor_duns_number
#' @param agency_name
#' @param contracting_office_name
#' @param contracting_agency_name
#' @param principal_naics_code
#' @param award_status
#' @param contract_type
#' @param type_of_contract_pricing contract pricing \itemize{
#' \item NULL
#' \item FIXED
#' \item NO FEE
#' \item RANGE - VARIES BY AMOUNT
#' \item RANGE - VARIES BY OTHER FACTOR
#' }
#' @param contract_id contract id
#' @param ref_idv_contract_id
#' @param ref_idv_agency_id
#' @param contracting_agency_id
#' @param contracting_office_id
#' @param funding_agency_id
#' @param funding_office_id
#' @param funding_office_name
#' @param agency_code
#' @param department_id
#' @param department_name
#' @param last_mod_date
#' @param last_modified_by
#' @param award_completion_date
#' @param created_date
#' @param signed_date
#' @param effective_date
#' @param estimated_completion_date
#' @param obligated_amount
#' @param ultimate_contract_value
#' @param created_by
#' @param description_of_requirement
#' @param reason_for_modification
#' @param legislative_mandates
#' @param local_area_set_aside
#' @param socio_economic_indicators
#' @param multiyear_contract
#' @param national_interest_code
#' @param product_or_service_code
#' @param vendor_address_city
#' @param vendor_congress_district_code
#' @param vendor_address_country_code
#' @param vendor_address_country_name
#' @param vendor_address_state_code
#' @param vendor_address_state_name
#' @param vendor_address_zip_code
#' @param parse_contracts
#' @param return_message
#' @param contract_type_description
#' @param parent_vendor_name
#' @param research
#' @param parent_duns_number
#' @param cancellation_date
#' @param destroy_date
#' @param final_invoice_paid_date
#' @param funded_through_date
#' @param last_modified_date
#' @param physical_completion_date
#' @param reveal_date
#' @param solicitation_issue_date
#' @param sys_last_modified_date
#' @param vendor_registration_date
#' @param vendor_renewal_date
#' @param base_exercised_options_value
#' @param performance_district_code
#' @param performance_country
#' @param performance_state_name
#' @param sort_item
#' @param sort_descending
#' @param ...
#' @param award_type
#' @param solicitation_procedure
#' @param subcontract_plan
#' @param current_contract_value
#' @param dollars_obligated
#' @param contract_value
#' @param fee_range_lower_value
#' @param fee_range_upper_value
#' @param fixed_fee_value
#' @param total_current_contract_value
#' @param total_dollars_obligated
#' @param total_non_government_value
#' @param total_ultimate_contract_value
#' @param contract_fiscal_year
#' @param extent_competed
#' @param number_of_offers_received
#' @param clean_entity_column
#' @param snake_names
#' @param keep_key_columns
#' @param unformat
#' @param exclude_bloat
#'
#' @return
#' @export
#'
#' @examples
#'fpds_atom(global_vendor_name = "PALANTIR TECHNOLOGIES", signed_date = c("2019-01-01", "2019-05-02"))
fpds_atom <-
function(global_vendor_name = NA,
vendor_name = NA,
parent_vendor_name = NA,
department_name = NA,
award_type = NA,
research = NA,
vendor_duns_number = NA,
parent_duns_number = NA,
vendor_doing_business_as_name = NA,
agency_name = NA,
contracting_office_name = NA,
contracting_agency_name = NA,
principal_naics_code = NA,
award_status = NA,
subcontract_plan = NA,
solicitation_procedure = NA,
contract_type = NA,
contract_type_description = NA,
type_of_contract_pricing = NA,
contract_id = NA,
ref_idv_contract_id = NA,
ref_idv_agency_id = NA,
contracting_agency_id = NA,
contracting_office_id = NA,
funding_agency_id = NA,
funding_office_id = NA,
funding_office_name = NA,
agency_code = NA,
department_id = NA,
last_mod_date = NA,
last_modified_by = NA,
award_completion_date = NA,
created_date = NA,
signed_date = NA,
effective_date = NA,
estimated_completion_date = NA,
cancellation_date = NA,
destroy_date = NA,
final_invoice_paid_date = NA,
funded_through_date = NA,
last_modified_date = NA,
physical_completion_date = NA,
reveal_date = NA,
solicitation_issue_date = NA,
sys_last_modified_date = NA,
vendor_registration_date = NA,
vendor_renewal_date = NA,
base_exercised_options_value = NA,
current_contract_value = NA,
dollars_obligated = NA,
contract_value = NA,
fee_range_lower_value = NA,
fee_range_upper_value = NA,
fixed_fee_value = NA,
obligated_amount = NA,
total_current_contract_value = NA,
total_dollars_obligated = NA,
total_non_government_value = NA,
total_ultimate_contract_value = NA,
ultimate_contract_value = NA,
contract_fiscal_year = NA,
created_by = NA,
description_of_requirement = NA,
reason_for_modification = NA,
legislative_mandates = NA,
local_area_set_aside = NA,
socio_economic_indicators = NA,
multiyear_contract = NA,
national_interest_code = NA,
product_or_service_code = NA,
performance_district_code = NA,
performance_country = NA,
performance_state_name = NA,
vendor_address_city = NA,
vendor_congress_district_code = NA,
vendor_address_country_code = NA,
vendor_address_country_name = NA,
vendor_address_state_code = NA,
vendor_address_state_name = NA,
vendor_address_zip_code = NA,
extent_competed = NA,
number_of_offers_received = NA,
sort_item = "Signed Date",
use_future = F,
show_progress = T,
clean_address = T,
clean_entity_column = T,
sort_descending = T,
parse_contracts = F,
snake_names = F,
keep_key_columns = F,
exclude_bloat = F,
unformat = T,
return_message = T,
...) {
options(warn = -1)
df_urls <-
.generate_fpds_advanced_queries(
vendor_doing_business_as_name = vendor_doing_business_as_name,
award_type = award_type,
solicitation_procedure = solicitation_procedure,
global_vendor_name = global_vendor_name,
vendor_name = vendor_name,
parent_vendor_name = parent_vendor_name,
vendor_duns_number = vendor_duns_number,
parent_duns_number = parent_duns_number,
agency_name = agency_name,
contracting_office_name = contracting_office_name,
contracting_agency_name = contracting_agency_name,
principal_naics_code = principal_naics_code,
award_status = award_status,
contract_type = contract_type,
contract_type_description = contract_type_description,
type_of_contract_pricing = type_of_contract_pricing,
contract_id = contract_id,
ref_idv_contract_id = ref_idv_contract_id,
ref_idv_agency_id = ref_idv_agency_id,
contracting_agency_id = contracting_agency_id,
contracting_office_id = contracting_office_id,
funding_agency_id = funding_agency_id,
funding_office_id = funding_office_id,
funding_office_name = funding_office_name,
agency_code = agency_code,
department_id = department_id,
department_name = department_name,
research = research,
last_mod_date = last_mod_date,
last_modified_by = last_modified_by,
award_completion_date = award_completion_date,
created_date = created_date,
signed_date = signed_date,
effective_date = effective_date,
estimated_completion_date = estimated_completion_date,
cancellation_date = cancellation_date,
destroy_date = destroy_date,
final_invoice_paid_date = final_invoice_paid_date,
funded_through_date = funded_through_date,
last_modified_date = last_modified_date,
physical_completion_date = physical_completion_date,
reveal_date = reveal_date,
solicitation_issue_date = solicitation_issue_date,
sys_last_modified_date = sys_last_modified_date,
vendor_registration_date = vendor_registration_date,
vendor_renewal_date = vendor_renewal_date,
base_exercised_options_value = base_exercised_options_value,
current_contract_value = current_contract_value,
dollars_obligated = dollars_obligated,
contract_value = contract_value,
fee_range_lower_value = fee_range_lower_value,
fee_range_upper_value = fee_range_upper_value,
fixed_fee_value = fixed_fee_value,
obligated_amount = obligated_amount,
total_current_contract_value = total_current_contract_value,
total_dollars_obligated = total_dollars_obligated,
total_non_government_value = total_non_government_value,
total_ultimate_contract_value = total_ultimate_contract_value,
ultimate_contract_value = ultimate_contract_value,
contract_fiscal_year = contract_fiscal_year,
created_by = created_by,
description_of_requirement = description_of_requirement,
reason_for_modification = reason_for_modification,
legislative_mandates = legislative_mandates,
local_area_set_aside = local_area_set_aside,
socio_economic_indicators = socio_economic_indicators,
multiyear_contract = multiyear_contract,
national_interest_code = national_interest_code,
product_or_service_code = product_or_service_code,
performance_district_code = performance_district_code,
performance_country = performance_country,
performance_state_name = performance_state_name,
vendor_address_city = vendor_address_city,
vendor_congress_district_code = vendor_congress_district_code,
vendor_address_country_code = vendor_address_country_code,
vendor_address_country_name = vendor_address_country_name,
vendor_address_state_code = vendor_address_state_code,
vendor_address_state_name = vendor_address_state_name,
vendor_address_zip_code = vendor_address_zip_code,
sort_item = sort_item,
sort_descending = sort_descending,
return_message = return_message,
subcontract_plan = subcontract_plan,
extent_competed = extent_competed,
number_of_offers_received = number_of_offers_received,
...
)
if (!parse_contracts) {
return(df_urls)
}
urls <- df_urls$urlFPDSAtom
parse_fpds_atom_urls_safe <-
possibly(parse_fpds_atom_urls, tibble())
all_data <-
parse_fpds_atom_urls(
urls = urls,
return_message = return_message,
unformat = unformat,
clean_entity_column = clean_entity_column,
keep_key_columns = keep_key_columns,
exclude_bloat = exclude_bloat,
use_future = use_future,
clean_address = clean_address,
snake_names = F,
)
if (nrow(all_data) == 0) {
"No Matches" %>% message()
return(invisible())
}
all_data <-
all_data %>%
left_join(df_urls, by = "urlFPDSAtom") %>%
.remove_na()
# all_data <-
# all_data %>%
# resolve_listed_duns() %>%
# add_department_codes()
#
if (snake_names) {
all_data <- all_data %>%
janitor::clean_names()
}
all_data
}
#' FPDS Free Text Term Count
#'
#' Searches FPDS for the number of contracts
#' for a vector of specified terms
#'
#' @param terms vector of terms
#' @param use_quote if \code{TRUE} uses quoted word
#' @param return_message if \code{TRUE} returns a message
#' @param clean_entity_column
#' @param snake_names
#'
#' @return
#' @export
#'
#' @examples
#' fpds_free_text_term_count(terms = c("Deep Learning", "Machine Learning"), use_quote = c(T,F))
fpds_free_text_term_count <-
function(terms = NULL,
use_quote = c(T, F),
clean_entity_column = T,
snake_names = F,
return_message = T) {
if (length(terms) == 0) {
stop("No terms entered")
}
data <-
fpds_free_text_search(
terms = terms,
use_quote = use_quote,
parse_contracts = F,
return_message = return_message
)
if (nrow(data) == 0) {
"No matches" %>% message()
}
data <-
data %>%
select(-urlFPDSAtom) %>%
distinct() %>%
arrange(desc(countResults))
if (snake_names) {
data <- data %>% janitor::clean_names()
}
data
}
#' Add Solictiation Groups for FPDS Data
#'
#' @param data a \code{tibble}
#'
#' @return
#' @export
#'
#' @examples
add_solicitation_group <-
function(data) {
if (!data %>% hasName("codeProductService")) {
return(data)
}
data <-
data %>%
mutate(
idSolicitationGroup =
case_when(
is.na(codeProductService) ~ NA_character_,
codeProductService %>% substr(1, 1) %>% str_detect("[A-Z]") ~ codeProductService %>% substr(1, 1),
TRUE ~ codeProductService %>% substr(1, 2)
)
)
df_psc <-
dictionary_psc_active() %>% distinct(idSolicitationGroup, nameSolicitationGroup)
data %>%
select(-matches("nameSolicitationGroup")) %>%
left_join(df_psc, by = "idSolicitationGroup")
}
# bulk --------------------------------------------------------------------
#' Bulk FPDS Download
#'
#' Download data from FPDS for specified
#' years
#'
#' @param years vector of years from 1970 until present
#' @param path file path
#' @param split_amounts if \code{TRUE} split amounts for search
#'
#' @return \code{tibble()}
#' @export
#'
#' @examples
bulk_download_fpds <-
function(years = 1977:2000,
path = "Desktop/data/usa_spending/fpds",
split_amounts = T,
return_message = F,
snake_names = T) {
oldwd <- getwd()
setwd("~")
years %>%
walk(function(year) {
year %>% message()
data <-
bulk_fpds_csv_years(
years = year,
split_amounts = split_amounts,
return_message = return_message
)
data_path <- glue("{path}/{year}.tsv.gz")
data <- data %>%
munge_lite(unformat = T, snake_names = snake_names)
data %>% write_csv(data_path)
rm(data)
gc()
return(invisible())
})
}
# bulk_atom ---------------------------------------------------------------
.atom_year_url_old <-
function(year = 1979, splits = 4) {
df_urls <-
year %>%
map_dfr(function(year) {
if (year == 1977) {
start <- "1900-01-01"
end <- "1977-12-31"
}
if (year != 1977) {
start <- glue("{year}-01-01")
end <- glue("{year}-12-31")
}
fpds_atom(signed_date = c(start, end)) %>%
mutate(yearData = year) %>%
select(yearData, everything())
})
matches <- nrow(df_urls) * 10 %>% formattable::comma(digits = 0)
glue("\n\n{red({matches})} procurement actions for years: {blue(year)}\n\n") %>% message()
splits <- case_when(year == 1977 ~ 4,
TRUE ~ splits)
df_urls <- df_urls %>%
mutate(group = yearData %>% ntile(n = splits))
df_urls
}
.atom_year_url <-
function(year = 2019, splits = 10) {
df_urls <-
year %>%
map_dfr(function(year) {
if (year == 1977) {
start <- "1900-01-01"
end <- "1977-12-31"
data <-
fpds_atom(signed_date = c(start, end)) %>%
mutate(yearData = year) %>%
select(yearData, everything())
return(data)
}
df_months <-
.year_month_normal(year = year)
all_data <-
1:nrow(df_months) %>%
map_dfr(function(x){
glue("Period {x} for {year}") %>% message()
df_m <- df_months[x,]
start <-df_m$dateStart
end <- df_m$dateEnd
data <-
fpds_atom(signed_date = c(start, end)) %>%
mutate(yearData = year) %>%
select(yearData, everything())
data
})
all_data
})
matches <- nrow(df_urls) * 10 %>% formattable::comma(digits = 0)
glue("\n\n{red({matches})} procurement actions for years: {blue(year)}\n\n") %>% message()
splits <- case_when(year == 1977 ~ 4,
TRUE ~ splits)
df_urls <-
df_urls %>%
mutate(group = yearData %>% ntile(n = splits))
df_urls
}
.consolidate_year_rda <-
function(path = "Desktop/data/usa_spending/fpds_atom",
year = 1978,
exclude_bloat = T) {
oldwd <- getwd()
setwd("~")
final_path <-
glue("{path}/{year}")
final_file <-
glue("{year}_final.rda")
setwd(final_path)
files <-
list.files()[list.files() %>% str_detect(".rda")] %>%
discard(function(x) {
x %>% str_detect("final")
})
data <-
files %>% map_dfr(read_rda)
names(data) <-
names(data) %>% str_replace_all("_office_contracting", "office_award")
add_vendor <- names(data)[names(data) %in% c(
"city",
"code_congressional_district",
"code_country",
"code_state",
"zipcode",
"name_state",
"address_street2",
"name_country",
"fax",
"telephone",
"address_street1",
"country"
)]
if (length(add_vendor) > 0) {
names(data)[names(data) %in% add_vendor] <-
names(data)[names(data) %in% add_vendor] %>% str_c("_vendor")
}
if (exclude_bloat) {
data <-
data %>%
select(-matches("url_fpds|names_vendor"))
}
data <- data %>%
mutate_at(data %>% select(matches("id_duns")) %>% names(),
list(function(x) {
if_else(x == 0, NA_real_, x)
}))
data <-
data %>%
mutate(is_contractor_not_vendor = name_contractor != name_vendor)
v_n <-
data %>% select(matches("name_vendor|name_contractor")) %>% names()
data <-
data %>%
mutate_at(v_n, list(function(x) {
if_else(x == "NO DATA FROM D AND B", NA_character_, x)
}))
data <- data %>% mutate_if(is.numeric, as.numeric)
### deal with counts
data %>% save(file = final_file)
files %>%
walk(function(file) {
if (file.exists(file))
#Delete file if it exists
file.remove(file)
})
if (getwd() != oldwd) {
setwd("~")
setwd(oldwd)
}
glue("\n\nFINISHED {crayon::red(year)}\n\n\n") %>% message()
return(invisible())
}
#' Consolidate bulk bulk .rda files
#'
#' @param path file path
#' @param years vector of sub-folder years
#' @param exclude_bloat if \code{TRUE} excludes bloat columns
#'
#' @return
#' @export
#'
#' @examples
consolidate_atom_years_rda <-
function(path = "Desktop/data/usa_spending/fpds_atom",
years = 1978,
exclude_bloat = T) {
years %>%
future_walk(function(year) {
.consolidate_year_rda(path = path,
year = year,
exclude_bloat = exclude_bloat)
}, .progress = T)
}
#' FPDS Atom URL Dictionaries
#'
#' @param years vector of years
#' @param splits number of splits
#'
#' @return
#' @export
#'
#' @examples
fpds_years_atom_urls <-
function(years = 1979:2020,
splits = 4) {
data <-
years %>%
map_dfr(function(year) {
.atom_year_url(year = year, splits)
})
data
}
.fpds_atom_year_bulk <-
function(year = 1977,
strategy = "multisession",
splits = 5,
start_split = NULL,
end_split = NULL,
clean_entity_column = F,
unformat = T,
keep_key_columns = F,
show_progress = T,
path = "Desktop/data/usa_spending/fpds_atom",
snake_names = T,
clean_address = T,
return_message = T) {
df_urls <-
fpds_years_atom_urls(years = year, splits = splits)
if (length(start_split) > 0) {
df_urls <-
df_urls %>%
filter(group >= start_split)
}
oldwd <- getwd()
setwd("~")
has_year <-
path %>% list.dirs() %>% str_detect(as.character(year)) %>% sum(na.rm = T) == 1
final_path <- glue("{path}/{year}") %>% as.character()
if (!has_year) {
dir.create(path = final_path)
}
first_group <- df_urls$group %>% min()
if (length(end_split) > 0) {
splits <- end_split
}
first_group:splits %>%
walk(function(x) {
urls <-
df_urls %>%
filter(group == x) %>%
pull(urlFPDSAtom)
x <- as.character(x)
num_slug <-
case_when(
nchar(x) == 1 ~ glue("000{x}") %>% as.character(),
nchar(x) == 2 ~ glue("00{x}") %>% as.character(),
nchar(x) == 3 ~ glue("0{x}") %>% as.character(),
TRUE ~ as.character(x)
)
file_name <-
glue("{year}_{num_slug}.rda") %>% as.character()
data_path <-
glue("{final_path}/{file_name}") %>% as.character()
if (length(strategy) > 0) {
plan(strategy = strategy)
}
data <-
parse_fpds_atom_urls(
urls = urls,
clean_entity_column = clean_entity_column,
unformat = unformat,
keep_key_columns = keep_key_columns,
return_message = return_message,
exclude_bloat = T,
clean_address = clean_address,
snake_names = F,
use_future = T,
show_progress = show_progress
)
if (snake_names) {
data <- data %>%
janitor::clean_names()
}
data %>% save(file = data_path)
closeAllConnections()
gc()
return(invisible())
})
if (getwd() != oldwd) {
setwd(oldwd)
}
message("FINISHED")
gc()
return(invisible())
}
#' Bulk FPDS Download
#'
#' @param years vector of years
#' @param path file path
#' @param clean_entity_column if \code{TRUE} cleans entity columns
#' @param unformat if \code{TRUE} removes formattable formats
#' @param snake_names snake the names
#' @param return_message if \code{TRUE} returns message
#' @param splits if \code{TRUE} number of file splits
#' @param start_split
#' @param keep_key_columns
#' @param strategy future strategy \itemize{
#' \item sequential
#' \item multisession
#' \item cluster
#' }
#' @param clean_address
#'
#' @return
#' @export
#'
#' @examples
bulk_download_fpds_atom <-
function(years = 2001,
splits = 15,
end_split = NULL,
clean_entity_column = F,
unformat = T,
strategy = "multisession",
clean_address = T,
show_progress = T,
start_split = NULL,
keep_key_columns = F,
path = "Desktop/data/usa_spending/fpds_atom",
snake_names = T,
return_message = T) {
years %>%
walk(function(year_no) {
.fpds_atom_year_bulk(
year = year_no,
splits = splits,
clean_entity_column = clean_entity_column,
unformat = unformat,
strategy = strategy,
start_split = start_split,
keep_key_columns = keep_key_columns,
show_progress = show_progress,
path = path,
snake_names = snake_names,
clean_address = clean_address,
return_message = return_message,
end_split = end_split
)
})
closeAllConnections()
message("FINISHED")
gc()
return(invisible())
}
R Package Documentation
Browse R Packages
We want your feedback!
Note that we can't provide technical support on individual packages. You should contact the package authors for that.
Embedding an R snippet on your website
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.