Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Submit feedback
Sign in
Toggle navigation
H
Harmonize_Scripts
Project overview
Project overview
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Uncover
Harmonize_Scripts
Commits
3e986eb3
Commit
3e986eb3
authored
Apr 13, 2023
by
GNajeral
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Update to local script to show the range in which failing variables lie
parent
2c02e92b
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
43 additions
and
15 deletions
+43
-15
valid_variables_script2.R
valid_variables_script2.R
+1
-1
valid_variables_script_local.R
valid_variables_script_local.R
+42
-14
No files found.
valid_variables_script2.R
View file @
3e986eb3
...
...
@@ -16,7 +16,7 @@ source("necessary_functions_connection.R")
dep_list
=
c
(
"jsonlite"
,
"stringr"
,
"DSI"
,
"DSOpal"
,
"DSLite"
,
"fields"
,
"metafor"
,
"ggplot2"
,
"gridExtra"
,
"data.table"
,
"dsBaseClient"
,
"openxlsx"
)
install_dependencies
(
dep_list
)
codebook_file
<-
"20220315_Data
Harmonisation.xlsb.xlsx"
codebook_file
<-
"20220315_Data
_
Harmonisation.xlsb.xlsx"
codebook_demo
<-
read.xlsx
(
codebook_file
,
sheet
=
2
)
codebook_com_and_rf
<-
read.xlsx
(
codebook_file
,
sheet
=
3
)
...
...
valid_variables_script_local.R
View file @
3e986eb3
...
...
@@ -2,6 +2,7 @@ rm(list=ls())
dir_name
<-
readline
(
"Introduce the name of the directory please: "
)
# C:\Users\guill\Documents\harmonize_scripts
# /Users/gnl/Documents/CTB UPM/UNCOVER/uncover_harmonization
setwd
(
dir_name
)
...
...
@@ -11,7 +12,7 @@ source("dependency_installer.R")
dep_list
=
c
(
"jsonlite"
,
"stringr"
,
"DSI"
,
"DSOpal"
,
"DSLite"
,
"fields"
,
"metafor"
,
"ggplot2"
,
"gridExtra"
,
"data.table"
,
"dsBaseClient"
,
"openxlsx"
)
install_dependencies
(
dep_list
)
codebook_file
<-
"20220315_Data
Harmonisation.xlsb.xlsx"
codebook_file
<-
"20220315_Data
_
Harmonisation.xlsb.xlsx"
codebook_demo
<-
read.xlsx
(
codebook_file
,
sheet
=
2
)
codebook_com_and_rf
<-
read.xlsx
(
codebook_file
,
sheet
=
3
)
...
...
@@ -101,11 +102,13 @@ check_valid_name <- function(col_name){
}
check_valid_values_continuous
<-
function
(
colname
,
codebook_param
,
column
){
column
<-
as.numeric
(
column
[
column
!=
"."
])
possible_values_format
<-
codebook_param
$
Possible.values.format
[
codebook_param
$
Harmonised.variable.name
==
colname
]
value_format
<-
strsplit
(
possible_values_format
,
" / "
)[[
1
]]
min_value
<-
str_trim
(
gsub
(
","
,
"."
,
(
sub
(
"-.*"
,
""
,
value_format
[
1
]))))
max_value
<-
str_trim
(
gsub
(
","
,
"."
,
(
sub
(
".*-"
,
""
,
value_format
[
1
]))))
if
(
min_value
==
""
){
...
...
@@ -114,9 +117,6 @@ check_valid_name <- function(col_name){
}
min_value
<-
as.double
(
min_value
)
max_value
<-
as.double
(
max_value
)
print
(
colname
)
print
(
min_value
)
print
(
max_value
)
failing_values
<-
column
[
column
<
min_value
|
column
>
max_value
]
number_of_failing_values
<-
length
(
failing_values
[
!
is.na
(
failing_values
)])
...
...
@@ -128,15 +128,36 @@ check_valid_name <- function(col_name){
failing_values
<-
failing_values
[
!
is.na
(
failing_values
)]
failing_value_counts
<-
table
(
failing_values
)
str_res
<-
paste
(
colname
,
"has failing values:"
)
for
(
i
in
seq_along
(
failing_value_counts
))
{
value
<-
names
(
failing_value_counts
)[
i
]
count
<-
failing_value_counts
[
i
]
str_res
<-
paste
(
str_res
,
paste
(
value
,
"("
,
count
,
"times)"
,
collapse
=
" "
),
sep
=
" "
)
}
str_res
<-
paste
(
str_res
,
"should be in range"
,
value_format
,
"(continuous)"
,
sep
=
" "
)
str_res
<-
paste
(
colname
,
"has wrong values:"
)
str_res
<-
paste
(
str_res
,
paste
(
number_of_failing_values
,
","
,
sep
=
""
))
str_res
<-
paste
(
str_res
,
"should be in the range:"
,
paste
(
min_value
,
max_value
,
sep
=
"-"
),
"(continuous),"
,
sep
=
" "
)
first
<-
names
(
failing_value_counts
)[
1
]
last
<-
names
(
failing_value_counts
)[
length
(
failing_value_counts
)]
print
(
colname
)
print
(
"first"
)
print
(
first
)
print
(
"last"
)
print
(
last
)
str_res
<-
paste
(
str_res
,
"Values lie in the range:"
,
paste
(
"["
,
first
,
" - "
,
last
,
"]"
,
sep
=
""
))
# if(number_of_failing_values < 30) {
# # list_values <- paste( failing_values , collapse = " ")
# str_res <- paste(str_res, "Wrong values:")
#
# for (i in seq_along(failing_value_counts)) {
# value <- names(failing_value_counts)[i]
# count <- failing_value_counts[i]
# str_res <- paste(str_res, paste(value, paste("(", count, ")", sep = ""), collapse = " "), sep = " ")
# }
# }
}
return
(
str_res
)
...
...
@@ -324,6 +345,13 @@ check_valid_values <- function(valid_colnames, codebook_param){
next
}
if
(
is.na
(
codebook_param
$
Possible.values.format
[
codebook_param
$
Harmonised.variable.name
==
name
])){
variable
<-
paste
(
"Variable "
,
name
,
" with no value format"
,
sep
=
" "
)
res
<-
paste
(
res
,
variable
,
sep
=
"\n"
)
next
}
result
=
switch
(
column_type
,
"Continuous"
=
check_valid_values_continuous
(
name
,
codebook_param
,
column
),
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment