cxq2spss_0_1_1/Qfetch.pm0100644000241400002010000000355210337334123013615 0ustar c12902evalpackage Qfetch; # Qfetch adopted from XIMS (t/lib/XIMS/Test.pm) # s. http://xims.info/ use strict; use LWP::UserAgent; use HTTP::Cookies; ################################## # some helpful subs ################################## my $found_cookie; sub new { my $class = shift; my ( $timeout ) = @_; $timeout ||= 10; my %self = ( UA => LWP::UserAgent->new( requests_redirectable => ['GET','POST'], cookie_jar => HTTP::Cookies->new(), ) ); $self{UA}->agent("QFetcher/0.1 "); $self{UA}->timeout($timeout); return bless \%self, $class; } sub login { my $self = shift; my ( $user, $pass, $http_host ) = @_; my $url = $http_host . '/goxims/user'; # warn "using: " . $user . ' ' . $pass . ' ' . $url; my $req = HTTP::Request->new(POST => $url); $req->content_type('application/x-www-form-urlencoded'); $req->content("dologin=1&userid=$user&password=$pass"); # Pass request to the user agent and get a response back my $res = $self->{UA}->request($req); $res = $self->{UA}->request($req); my $cookie_parser = $self->{UA}->cookie_jar(); $cookie_parser->extract_cookies(); $cookie_parser->scan( \&cookie_scan ); if ( length( $found_cookie ) > 0 ) { # warn "user $user logged in\n"; $self->{Cookie} = $cookie_parser; $self->{session_id} = $found_cookie; $self->{logged_in} = 1; return $res; } return undef; } sub get { my $self = shift; my ( $uri, $http_host) = @_; $uri = $http_host . $uri; # warn "getting $uri \n"; my $req = HTTP::Request->new(GET => $uri); $self->{Cookie}->add_cookie_header( $req ); my $res = $self->{UA}->request( $req ); return $res; } sub cookie_scan { $found_cookie = $_[2] if $_[1] eq 'session'; } 1; cxq2spss_0_1_1/README0100644000241400002010000000722210337334015012723 0ustar c12902evalcxq2spss - Convert XIMS-Questionnaire Results to SPSS Input Data Copyright (C) 2004-2005 by Gregor Retti Introduction The result data of a XIMS-questionnaire is not usable in SPSS. A XIMS-questionnaire consists of XML-data making up questions and answers, and result data, which is stored in a database table. The result data may be exported in different formats. The format in question for converting the result data to SPSS input data is 'Raw result data (tab-separated)'. cxq2spss converts the result data to a format usable as input data for SPSS. Installation cxq2spss consists of two programs and one module: - cxq2spss-fetch.pl To fetch the XML-questionnaire part from a XIMS-server. - cxq2spss-process.pl To convert the data. - Qfetch.pm A module adopted from XIMS and used by cxq2spss-fetch.pl cxq2spss uses several modules which can be found at CPAN. Most of the modules are already installed, if you run cxq2spss on the same machine as XIMS. You may just have to install Term::ReadPassword additionally. Step by Step Description 1) Get the 'Raw result data (tab-separated)' for you questionnaire, unzip it and put it into a convenient place on. 2) Run cxq2spss-fetch.pl and provide the following params: -u username -h host -f full path to file You may want to copy the 'full path to file' from your web-browser. cxq2spss-fetch.pl will try to find and extract the XML-questionnaire data from the XML returned by the host and write it to STDOUT. Redirect the output to a convenient place. 3) Set up a control file for the questionnaire data The control file holds information for each question/answer(s) in the questionnaire. As well it determines the order of the data in the output file. 3a) For questions which allow only one answer you need just one line answer-id [tab] variable-name Names of variable may be choosen without restriction. They are not really needed but allow for checking the output as a spreadsheet. You may want to pick short and meaningful names anyway. The names may be made up of characters, numbers and underscores. Names starting with underscores are reserved for multiple choices questions (s. below). 3b) For multiple choice questions you need one line for each answer answer-id underscore number-of-choice [tab] variable-name and - at the end of the control file - an entry for the 'whole' answer: answer-id [tab] variable-name-starting-with-underscore [tab] starting-position-of-first-answer Before the entries for the 'whole' multiple choice answers you should put the line: 0 [tab] STOP ... just to be sure. Here is an example of a control file with two multiple choice questions: 1.1 p_wissen 2.1_1 aspekt_1 2.1_2 aspekt_2 2.1_3 aspekt_3 2.1_4 aspekt_4 2.1_5 aspekt_5 2.1_6 aspekt_6 3.1 o_wissen 4.1 besprech 5.1 r_labor 6.1 w_quelle 7.1_1 beitr_1 7.1_2 beitr_2 7.1_3 beitr_3 7.1_4 beitr_4 7.1_5 beitr_5 7.1_6 beitr_6 7.1_7 beitr_7 7.1_8 beitr_8 7.1_9 beitr_9 7.1_10 beitr_10 7.1_11 beitr_11 7.1_12 beitr_12 8.1 beurteil 9.1 beteilig 10.1 verstoss 13.1.1 geschl 13.2.1 stellung 13.3.1 bereich 0 STOP 2.1 _aspekt 2 7.1 _beitrag 12 4) Run cxq2spss-process.pl with the following params: -x xml questionnaire -v spss variables file -f raw data file [-n null value] The 'null value' param controls which value will be used, if the question has not be answered. The default is 99. The result will be written to STDOUT. 5) Possible problems - cxq2spss-process.pl uses the unix sort command. If it is not in /bin you may have to change the path. - If your questionnaire contains non-ASCII-characters there may be a problem matching the XML-questionnaire data against the result data. You may have to play around with charset conversion to solve this problem. cxq2spss_0_1_1/cxq2spss-fetch.pl0100755000241400002010000000363610337334140015264 0ustar c12902eval#!/usr/bin/perl # # cxq2spss - Convert XIMS-Questionnaire Results to SPSS Input Data # # Copyright (C) 2004-2005 by Gregor Retti # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 1, or (at your option) # any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. # # v. 0.0.1 2004/08/07 # # v. 0.0.2 2005/11/06 # add commandline params use Qfetch; use strict; use Getopt::Std; use Term::ReadPassword; our(%opts); getopt('uhf', \%opts); die "Usage: $0 -u username -h host -f full path to file\n" unless $opts{u} && $opts{h} && $opts{f}; my $pass = read_password('Password: ',0,1); my $user = $opts{u}; my $host = 'http://'.$opts{h}; my $path = $opts{f}; $path .= '?passthru=1'; my $fetcher = Qfetch->new(); my $response = $fetcher->login($user, $pass, $host); $response = undef; $response = $fetcher->get($path, $host); if ($response->is_success) { print STDERR "Got data from $host for $path\n"; } else { die "Got no answer from $host for $path\n"; } use XML::LibXML; my $parser = XML::LibXML->new(); my $doc = $parser->parse_string( $response->content ); my $encoding = $doc->encoding(); my $quest = $doc->findnodes('/document/context/object/body/questionnaire')->get_node(1); if ($@) { die "No questionnaire in data: $@\n"; } my $outdoc = XML::LibXML::Document->new('1.0',$encoding); $outdoc->setDocumentElement($quest); print $outdoc->toString(0); cxq2spss_0_1_1/cxq2spss-process.pl0100755000241400002010000001460510337334154015654 0ustar c12902eval#!/usr/bin/perl # # cxq2spss - Convert XIMS-Questionnaire Results to SPSS Input Data # # Copyright (C) 2004-2005 by Gregor Retti # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 1, or (at your option) # any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. # # # v. 0.0.1 2004/08/07 # # v. 0.0.2 2005/11/06 # convert xml-questionnaire, raw results, and spss variable table # to spss output # # A note about umlauts & friends: XML::LibXML seems to convert to Latin-1 # so we add a function to convert raw data, which is UTF8, to Latin-1 # # v. 0.0.3 2005/11/07 # add 0 position at beginning of output data # add 0 line for unanswered questions (not supplied in raw data!) # # v. 0.1.0 2005/11/07 # old output not really useable w/ SPSS, so we re-write the thing # make a matrix of tans and spss-vars, multichoice answers will be # appended w/ _n # # v. 0.1.1 2005/11/14 # change null-values to $opts{n} (99 if non given) # - for single choice # - for multiple choice if no choice is selected use strict; use Getopt::Std; use Text::Iconv; our(%opts,%run); getopt('xvfn', \%opts); die "Usage: $0 -x xml questionnaire -v spss variables file -f raw data file [-n null value]\n" unless $opts{x} && $opts{v} && $opts{f}; my $filename = $opts{x}; my $spssvars = $opts{v}; my $rawdata = $opts{f}; my $nullvalue = $opts{n}; $nullvalue = 99 unless $nullvalue; my $sort = '/bin/sort --key=2'; use XML::LibXML; # read in spss vars in %spss open (SPSS, "< $spssvars") || die "Couldn't open $spssvars: $! \n"; my(%spss,@spss_tmp,$i); $i=1; while () { chop; @spss_tmp = split(/\t/,$_); $spss{$spss_tmp[0]}->{name} = $spss_tmp[1]; $spss{$spss_tmp[0]}->{number} = $i; $spss{maxvars} = $i if $spss_tmp[0] && !($spss_tmp[1] =~ /^_/); if ($spss_tmp[1] =~ /^_/) { $spss{$spss_tmp[0]}->{start_m} = $spss_tmp[2]; } $i++; } close SPSS; my($parser) = XML::LibXML->new(); $parser->validation(0); my $doc = $parser->parse_file($filename); my ($a_type,$a_id,$a_titles,$a_title_string,$j); # get the answers and build up mapping for spss vars my $answers = $doc->findnodes('//answer'); foreach my $answer ($answers->get_nodelist()) { $a_type = $answer->getAttribute('type'); next if $a_type ne 'Radio' && $a_type ne 'Checkbox' && $a_type ne 'Select'; $a_id = $answer->getAttribute('id'); if ( $spss{$a_id}->{name} ) { $run{$a_id}->{spssvar} = $spss{$a_id}->{name}; $run{$a_id}->{type} = $a_type; } else { next; } $a_titles = $answer->findnodes('./title'); if ($a_type eq 'Radio' || $a_type eq 'Select') { $j = 1; foreach my $a_title ($a_titles->get_nodelist()) { $a_title_string = $a_title->textContent; $run{$a_id}->{$a_title_string} = $j; $j++; } # max values is $j - 1 $run{$a_id}->{max_values} = $j - 1; } else { # it's a multivalue entry, so store the varname w/ the string $j = 1; foreach my $a_title ($a_titles->get_nodelist()) { $a_title_string = $a_title->textContent; $run{$a_id}->{$a_title_string} = $a_id.'_'.$j; $run{$a_id.'_'.$j}->{spssvar} = $spss{$a_id.'_'.$j}->{name}; $j++; } # max values is $j - 1 as we need to test for the missing answers $run{$a_id}->{max_values} = $j - 1; $spss{$a_id}->{end_m} = $spss{$a_id}->{start_m} + $run{$a_id}->{max_values}; } } # get the raw data and prepare output # data is: # 0 = id # 1 = tan # 2 = answer_id # 3 = text # 4 = timestamp open (DATA, "$sort $rawdata |") || die "Couldn't open $sort $rawdata: $! \n"; my(@data,$result,$k,$old_answer,$old_tan,%tans,@output); $old_answer = ''; $old_tan = ''; # set up header line w/ variable names $output[0] = 'id'; foreach my $key (keys %spss) { next unless ref $spss{$key}; if ($spss{$key}->{number}) { next if $spss{$key}->{number} > $spss{maxvars}; $output[$spss{$key}->{number}] = $spss{$key}->{name}; } } print_collected_data(@output); my($old_type,$start_m,$end_m,$data_m,@data_m); while () { chop; @data = split(/\t/,$_); $tans{$data[1]}->{$data[2]} = 1; # check and fill multiple choice questions if ($old_type ne $run{$data[2]}->{type}) { if ($old_type eq 'Checkbox') { # if old_type = Checkbox than we have to check and fill the values if ($data_m) { my($d); for ($d = $start_m; $d <= $end_m; $d++) { $output[$d] = $data_m[$d]; $output[$d] = 0 unless $output[$d]; } } $data_m = 0; @data_m = (); $start_m = 0; $end_m = 0; } $old_type = $run{$data[2]}->{type} } if ($old_tan ne $data[1]) { # new tan, print output if (defined $tans{$old_tan}) { for ($j=1; $j <= $spss{maxvars}; $j++) { next if $output[$j] eq '0'; $output[$j] = $nullvalue unless $output[$j]; } print_collected_data(@output); } $old_tan = $data[1]; @output = (); $output[0] = $data[1]; # store the tan } if ($run{$data[2]}) { $data[3] = convert_utf8_iso_latin_1($data[3]); # for types Radio and Select if ($run{$data[2]}->{type} eq 'Radio' || $run{$data[2]}->{type} eq 'Select') { for ($k=1; $k <= $run{$data[2]}->{max_values}; $k++) { if ($run{$data[2]}->{$data[3]} eq $k) { $output[$spss{$data[2]}->{number}] = $k; } } } elsif ( $run{$data[2]}->{type} eq 'Checkbox' ) { $data_m = 1; $start_m = $spss{$data[2]}->{start_m}; $end_m = $spss{$data[2]}->{end_m}; my $a_id = $run{$data[2]}->{$data[3]}; $data_m[$spss{$a_id}->{number}] = 1; } } } # last line is header from raw export after sort ;-) sub print_collected_data { print join("\t",@_)."\n"; # print join(",",@_)."\n"; } # convert from UTF-8 sub convert_charset { my $input = shift; my $from = shift; my $to = shift; my $saved = $input; return $input unless ($from && $to); my $converter = Text::Iconv->new( $from, $to ); $input = $converter->convert($input) if defined $input; return $saved unless defined $converter->retval; return $input; } sub convert_utf8_iso_latin_1 { my $input = shift; return convert_charset($input,'UTF-8','ISO-8859-1'); } __END__