Fast Sparse Regression and Classification
! %
"
# &
$ (
$ '
)
% &
" *
" *
&
" +
(
" * (
" * *
,
"
)
$
)
-
( (
# ( " * "
(
.
"
" $
( /01 ( +
/
"
"
2
1 3
2
& 3
+
(
( (
" "
+
4 (
/ 1 (
"
*
/2
531 /61
"
(
( "
&
"
7
%
) /81
9
"
) 5 /:1
;
(
(
< "
(
0
# =86 : / (>
(
1
& /61
(
" /?1 5 (
*
"
&
$
/61
"
/01 /@1
&
(
/ 1
A
&
5
&
/@1 / 1 (
"
(
B
+
& /
1 +
"
'
&
(
/@1 & #
(
( "
/ 1
( /@1
"
+ /=1 " (
(
(
(
(
* "
* " 1 (
/"
%
(
" /@1 & &
(
* & '
&
"
A
"
* /01
&
( " /=1
/01
/61 / 1 C " ( / 1(
"
(
(
* % "
"
( (
+ ( /=1
( ( /0 1
&
(
/=1 /
/ /=1 & /0 1 " ( /
+
(
1 D "
(
1)
"
& +
1& /0 1
(
" (
(
( )
(
"
(
(
( +
* (
"
/?1
" & 5 /61 /001
+
& "
(
( /0 1 /061
&
& ( +
5 /61 "
& /61
(
*
(
"
(
$
# )"
+
) /
&
1 +
+ (
/
1
5
/61 + /061
5 )"
"
/061 "
"
/ 1
$ 3
/2 )(
1
E" (
/?1 /001
A(
/0 1 '
& "
+
/01 &
'
(
5 /081
&
/01 "
+
/0 1 +
&
/?1
(
"
. "
(
&
&
"
5
(
+
&
&
(
+
F 5
(
&
(
& (
(
(
F
( "
" /0:1
&
A( ( A(
(
" F
" "
/081
( (
&
*
'
(
& 5 & ( 01
/
( ( "
( ( (
+ /0?1 /0@1 +
(
2
$
3/ 6
5
0==61
.
(
(
&
( /0 1
+
)
0
0.0
0.0
0.2
0.1
0.2
Model error
0.6 0.4
Model error
0.3
0.8
0.4
1.0
Corr = 0.0
step
0.0
0 , A " * "
0.1
0.2
0.5
( E7
lasso
&
elnet 1.5
step
" " * * &
:
0.0
0.1
06, A " * " : !
0.2
0.5
( E7
&
" " * *
( /
(
(
! /8 1
1 + "
( & &
&
& &
&
elnet 1.5
"
0 " "
lasso
D*
(
E7 (
0@
" &
"
" 00
& /8 1 (
(
(
( 0
"
E7
&
"
+ 0 :
" "
( & + E7
08 #
& 0 :
(
&
" 0:
!
"
( /
!
" !
&
/
1
1
&
0 ( & ( &
" + #
"
( /
1
( /8 1 &
# &
"
( &
"
&
(
E7
+
(
+ %
" ) ' ! &
&
0 E7 (
( (
+ (
/
&
0 ( "
( ( 5 ' " % "
* /!"
( %
(
!
1
(
"
& "
B
&
"
&
+
! &
)
(
(
/
( (
& 1
(/ " "
0: "
( !
+
&
(
" 0 )0
6
("
&
(
+ 0
" "
& /
&
+ 1
(
"
*
(
! /?1
A (0 0 0 !
( /0:1 (
*
!
A (
! &
B6 & &
& " " & 1
/
:
!
&
( & &
06 0:
"
+
" *
(
& ! "
"
A
( (
&
(
(
0
#
( 1
%
00 +
!
( 86
#
/8 1 "
' (
!"
+
"
(
00 06
"
( ) * ( & 1
&
(
/?1 + + * ! F &
!
. " /( "
" 6
5
&
&
1 +
&
( *
0
(
Corr = 0.4, a > 0
0.08 0.07
1 − AOC
0.10
0.06
0.15
1 − AOC
0.20 0.15
0.0
0.1
0.2
0.5
08, 0$#< " * " * *
lasso
(
0.04
0.05
0.05
0.05
0.10
1 − AOC
0.20
0.25
0.09
0.25
0.10
Corr = 0
0.30
Corr = 0.4
elnet 1.5
0.0
" E7 "
0.1
0.2
0.5
lasso
elnet 1.5
(
" E7 "
0:, 0$#< " * " * *
:
0.0
:
0.1
0.2
0.5
lasso
0?, 0 $ #< " * " * *
(
elnet 1.5
" E7 "
:
&
&
!
" & ! (
" D"
F
&
( 5
5
&
(
(
A
(
&
"
A (
/:1 .
(
(
"
( /
" % &
(
(
1
!
,- ) 08
#
0, Post
0.0
0.0
0.2
0.1
0.2
Model error
0.6 0.4
Model error
0.3
0.8
0.4
1.0
Corr = 0, Post
lasso
step
0.0
0.1
0=,
( 0@)
0.5
( (
0
0.2
lasso
elnet 1.5
0
step
&
0.0
,
0
0.2
0.5
( (
lasso
06
elnet 1.5
&
06
) * &
(
"
E7 * 0 )06
( /
0.1
1 0 )06 + (
* (
" * "
$ +
( ( &
+
&
/
(
E7
&
1
"
E7 &
0
$ "
"
/
8 01
0 /
001 ( / 0
" * :
! 1
"
" " + 0@ 0=1 & " *
&" "
& " *
" & " 1 & /
/
(
1 ( /
/
/8 1 '
&
" * 06 &
"
(
1
# &
(
" (
! "
( "
+
& &
H
(
0@)
&
"
5
+ $
/0?1 /0@1
&
0
( .
"
( 0) 6
&
(
/:1
E7
*
" *
(
)
D
"
"
08)0?
"
+
/
1
(
08)0? 08 E7 +
"
' ( 61
0 #< /801 " " * / & ! "
" &
( " " " *
+ (
E7 (
&
1
&" A
1 + /
&
" *
*
" *
%$"
/
@ 01 +
(
&
" */
" *
" *
/
$ $
" * A
61
(
&
&
%$"
2
&
+ 5 & ( & " /
? !
@
(
/?1 ( '
+ &
5
(
%
(
&
% /0@1
01 ( &
(
'
"
)(
)"
Corr = 0.4, Post
Corr = 0.4, a > 0, Post
0.08
1 − AOC
0.04
0.05
0.05
0.06
0.10
0.15
1 − AOC
0.20 0.15 0.10
1 − AOC
0.20
0.25
0.10
0.25
0.30
Corr = 0, Post
0.0
0.1
0.2
0.5
lasso
0,
elnet 1.5
(
0.0
08
0.5
lasso
elnet 1.5
(
0.0
0:
( 1
(
( 0= /
0
8
&
"
0.2
0.5
lasso
elnet 1.5
(
0?
&
08
+
0.1
6,
& (
$
0.2
,
&
/
0.1
0:
&
(
(
0?
)% " 0
:
1 +
*
(
+ (
& *
+ /
$
( +
*
)( 1 & $% / $ 1
)"
(
+
& &
"
:
(
+ &
* )(
+
)"
+
(
* +
&
/
$
1
& (
8
( &
"
(
(
8
&
0: ,-) &
( % +
(
(
:
(
,- ) (
( %
& :
/ .
"
+
5 (1 % &
& +
8
( (
(
(
8
(
$"
5
6N "
( :N J
5 ,
N(
? N(
J
(
( *
&
&
(
$"
+ +
! #
& 6
0
" " (
6
'
Squared−error, Corr = 0.0
20 15 10 0
5
Penalty frequency
0.6 0.4 0.2
Model error
0.8
Squared−error, Corr = 0.0
las−dir
las−sel
brg−dir
brg−sel
0.0
0.2
en1.5
6 2
4
Penalty frequency
0.15
8 10 12 14
Logistic, Corr = 0.0
0.25
Logistic, Corr = 0.0
0 las−sel
brg−dir
brg−sel
0.0
0.1
0.2
Method
8,
0: +
(
$" &
+
&
+ : 5 (
(
&
+
( ( ( % "
(
( /
" %
E7 *
/
$
/ &
(
(
& (
&
)(
)( * ( ( " $"
& 1
% )(
& ( *
(
0 :
$ 0:
5
8( (
( &
" #
* $
*
&
&
# &
$
$"
8
(
"
# &
$ (
$ '
)
% &
" *
" *
&
" +
(
" * (
" * *
,
"
)
$
)
-
( (
# ( " * "
(
.
"
" $
( /01 ( +
/
"
"
2
1 3
2
& 3
+
(
( (
" "
+
4 (
/ 1 (
"
*
/2
531 /61
"
(
( "
&
"
7
%
) /81
9
"
) 5 /:1
;
(
(
< "
(
0
# =86 : / (>
(
1
& /61
(
" /?1 5 (
*
"
&
$
/61
"
/01 /@1
&
(
/ 1
A
&
5
&
/@1 / 1 (
"
(
B
+
& /
1 +
"
'
&
(
/@1 & #
(
( "
/ 1
( /@1
"
+ /=1 " (
(
(
(
(
* "
* " 1 (
/"
%
(
" /@1 & &
(
* & '
&
"
A
"
* /01
&
( " /=1
/01
/61 / 1 C " ( / 1(
"
(
(
* % "
"
( (
+ ( /=1
( ( /0 1
&
(
/=1 /
/ /=1 & /0 1 " ( /
+
(
1 D "
(
1)
"
& +
1& /0 1
(
" (
(
( )
(
"
(
(
( +
* (
"
/?1
" & 5 /61 /001
+
& "
(
( /0 1 /061
&
& ( +
5 /61 "
& /61
(
*
(
"
(
$
# )"
+
) /
&
1 +
+ (
/
1
5
/61 + /061
5 )"
"
/061 "
"
/ 1
$ 3
/2 )(
1
E" (
/?1 /001
A(
/0 1 '
& "
+
/01 &
'
(
5 /081
&
/01 "
+
/0 1 +
&
/?1
(
"
. "
(
&
&
"
5
(
+
&
&
(
+
F 5
(
&
(
& (
(
(
F
( "
" /0:1
&
A( ( A(
(
" F
" "
/081
( (
&
*
'
(
& 5 & ( 01
/
( ( "
( ( (
+ /0?1 /0@1 +
(
2
$
3/ 6
5
0==61
.
(
(
&
( /0 1
+
)
0
0.0
0.0
0.2
0.1
0.2
Model error
0.6 0.4
Model error
0.3
0.8
0.4
1.0
Corr = 0.0
step
0.0
0 , A " * "
0.1
0.2
0.5
( E7
lasso
&
elnet 1.5
step
" " * * &
:
0.0
0.1
06, A " * " : !
0.2
0.5
( E7
&
" " * *
( /
(
(
! /8 1
1 + "
( & &
&
& &
&
elnet 1.5
"
0 " "
lasso
D*
(
E7 (
0@
" &
"
" 00
& /8 1 (
(
(
( 0
"
E7
&
"
+ 0 :
" "
( & + E7
08 #
& 0 :
(
&
" 0:
!
"
( /
!
" !
&
/
1
1
&
0 ( & ( &
" + #
"
( /
1
( /8 1 &
# &
"
( &
"
&
(
E7
+
(
+ %
" ) ' ! &
&
0 E7 (
( (
+ (
/
&
0 ( "
( ( 5 ' " % "
* /!"
( %
(
!
1
(
"
& "
B
&
"
&
+
! &
)
(
(
/
( (
& 1
(/ " "
0: "
( !
+
&
(
" 0 )0
6
("
&
(
+ 0
" "
& /
&
+ 1
(
"
*
(
! /?1
A (0 0 0 !
( /0:1 (
*
!
A (
! &
B6 & &
& " " & 1
/
:
!
&
( & &
06 0:
"
+
" *
(
& ! "
"
A
( (
&
(
(
0
#
( 1
%
00 +
!
( 86
#
/8 1 "
' (
!"
+
"
(
00 06
"
( ) * ( & 1
&
(
/?1 + + * ! F &
!
. " /( "
" 6
5
&
&
1 +
&
( *
0
(
Corr = 0.4, a > 0
0.08 0.07
1 − AOC
0.10
0.06
0.15
1 − AOC
0.20 0.15
0.0
0.1
0.2
0.5
08, 0$#< " * " * *
lasso
(
0.04
0.05
0.05
0.05
0.10
1 − AOC
0.20
0.25
0.09
0.25
0.10
Corr = 0
0.30
Corr = 0.4
elnet 1.5
0.0
" E7 "
0.1
0.2
0.5
lasso
elnet 1.5
(
" E7 "
0:, 0$#< " * " * *
:
0.0
:
0.1
0.2
0.5
lasso
0?, 0 $ #< " * " * *
(
elnet 1.5
" E7 "
:
&
&
!
" & ! (
" D"
F
&
( 5
5
&
(
(
A
(
&
"
A (
/:1 .
(
(
"
( /
" % &
(
(
1
!
,- ) 08
#
0, Post
0.0
0.0
0.2
0.1
0.2
Model error
0.6 0.4
Model error
0.3
0.8
0.4
1.0
Corr = 0, Post
lasso
step
0.0
0.1
0=,
( 0@)
0.5
( (
0
0.2
lasso
elnet 1.5
0
step
&
0.0
,
0
0.2
0.5
( (
lasso
06
elnet 1.5
&
06
) * &
(
"
E7 * 0 )06
( /
0.1
1 0 )06 + (
* (
" * "
$ +
( ( &
+
&
/
(
E7
&
1
"
E7 &
0
$ "
"
/
8 01
0 /
001 ( / 0
" * :
! 1
"
" " + 0@ 0=1 & " *
&" "
& " *
" & " 1 & /
/
(
1 ( /
/
/8 1 '
&
" * 06 &
"
(
1
# &
(
" (
! "
( "
+
& &
H
(
0@)
&
"
5
+ $
/0?1 /0@1
&
0
( .
"
( 0) 6
&
(
/:1
E7
*
" *
(
)
D
"
"
08)0?
"
+
/
1
(
08)0? 08 E7 +
"
' ( 61
0 #< /801 " " * / & ! "
" &
( " " " *
+ (
E7 (
&
1
&" A
1 + /
&
" *
*
" *
%$"
/
@ 01 +
(
&
" */
" *
" *
/
$ $
" * A
61
(
&
&
%$"
2
&
+ 5 & ( & " /
? !
@
(
/?1 ( '
+ &
5
(
%
(
&
% /0@1
01 ( &
(
'
"
)(
)"
Corr = 0.4, Post
Corr = 0.4, a > 0, Post
0.08
1 − AOC
0.04
0.05
0.05
0.06
0.10
0.15
1 − AOC
0.20 0.15 0.10
1 − AOC
0.20
0.25
0.10
0.25
0.30
Corr = 0, Post
0.0
0.1
0.2
0.5
lasso
0,
elnet 1.5
(
0.0
08
0.5
lasso
elnet 1.5
(
0.0
0:
( 1
(
( 0= /
0
8
&
"
0.2
0.5
lasso
elnet 1.5
(
0?
&
08
+
0.1
6,
& (
$
0.2
,
&
/
0.1
0:
&
(
(
0?
)% " 0
:
1 +
*
(
+ (
& *
+ /
$
( +
*
)( 1 & $% / $ 1
)"
(
+
& &
"
:
(
+ &
* )(
+
)"
+
(
* +
&
/
$
1
& (
8
( &
"
(
(
8
&
0: ,-) &
( % +
(
(
:
(
,- ) (
( %
& :
/ .
"
+
5 (1 % &
& +
8
( (
(
(
8
(
$"
5
6N "
( :N J
5 ,
N(
? N(
J
(
( *
&
&
(
$"
+ +
! #
& 6
0
" " (
6
'
Squared−error, Corr = 0.0
20 15 10 0
5
Penalty frequency
0.6 0.4 0.2
Model error
0.8
Squared−error, Corr = 0.0
las−dir
las−sel
brg−dir
brg−sel
0.0
0.2
en1.5
6 2
4
Penalty frequency
0.15
8 10 12 14
Logistic, Corr = 0.0
0.25
Logistic, Corr = 0.0
0 las−sel
brg−dir
brg−sel
0.0
0.1
0.2
Method
8,
0: +
(
$" &
+
&
+ : 5 (
(
&
+
( ( ( % "
(
( /
" %
E7 *
/
$
/ &
(
(
& (
&
)(
)( * ( ( " $"
& 1
% )(
& ( *
(
0 :
$ 0:
5
8( (
( &
" #
* $
*
&
&
# &
$
$"
8
(
